Compare commits

..
Author SHA1 Message Date
John Ericson a4b62a646a nlohmann::json instance and JSON Schema for MemorySourceAccessor
As documented, this for file system objects themselves, since
`MemorySourceAccessor` is an implementation detail.
2025-10-21 13:26:16 -04:00
136 changed files with 1185 additions and 2701 deletions

No files matched your search

-14
View File
@@ -1,14 +0,0 @@
# Disable CodeRabbit auto-review to prevent verbose comments on PRs.
# When enabled: false, CodeRabbit won't attempt reviews and won't post
# "Review skipped" or other automated comments.
reviews:
auto_review:
enabled: false
review_status: false
high_level_summary: false
poem: false
sequence_diagrams: false
changed_files_summary: false
tools:
github-checks:
enabled: false
+1 -12
View File
@@ -16,17 +16,13 @@ inputs:
install_url:
description: "URL of the Nix installer"
required: false
default: "https://releases.nixos.org/nix/nix-2.32.1/install"
default: "https://releases.nixos.org/nix/nix-2.30.2/install"
tarball_url:
description: "URL of the Nix tarball to use with the experimental installer"
required: false
github_token:
description: "Github token"
required: true
use_cache:
description: "Whether to setup magic-nix-cache"
default: true
required: false
runs:
using: "composite"
steps:
@@ -122,10 +118,3 @@ runs:
source-url: ${{ inputs.experimental-installer-version != 'latest' && 'https://artifacts.nixos.org/experimental-installer/tag/${{ inputs.experimental-installer-version }}/${{ env.EXPERIMENTAL_INSTALLER_ARTIFACT }}' || '' }}
nix-package-url: ${{ inputs.dogfood == 'true' && steps.download-nix-installer.outputs.tarball-path || (inputs.tarball_url || '') }}
extra-conf: ${{ inputs.extra_nix_config }}
- uses: DeterminateSystems/magic-nix-cache-action@565684385bcd71bad329742eefe8d12f2e765b39 # v13
if: ${{ inputs.use_cache == 'true' }}
with:
diagnostic-endpoint: ''
use-flakehub: false
use-gha-cache: true
source-revision: 92d9581367be2233c2d5714a2640e1339f4087d8 # main
+10 -12
View File
@@ -14,10 +14,6 @@ on:
default: true
type: boolean
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions: read-all
jobs:
@@ -33,7 +29,6 @@ jobs:
extra_nix_config:
experimental-features = nix-command flakes
github_token: ${{ secrets.GITHUB_TOKEN }}
use_cache: false
- run: nix flake show --all-systems --json
pre-commit-checks:
@@ -46,6 +41,7 @@ jobs:
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
extra_nix_config: experimental-features = nix-command flakes
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: DeterminateSystems/magic-nix-cache-action@main
- run: ./ci/gha/tests/pre-commit-checks
basic-checks:
@@ -96,6 +92,7 @@ jobs:
dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }}
# The sandbox would otherwise be disabled by default on Darwin
extra_nix_config: "sandbox = true"
- uses: DeterminateSystems/magic-nix-cache-action@main
# Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user:
# https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces
- run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
@@ -174,7 +171,7 @@ jobs:
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@c134e4c9e34bac6cab09cf239815f9339aaaf84e # v31.5.1
- uses: cachix/install-nix-action@v31
if: ${{ !matrix.experimental-installer }}
with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
@@ -230,13 +227,12 @@ jobs:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- uses: ./.github/actions/install-nix-action
- uses: cachix/install-nix-action@v31
with:
dogfood: false
extra_nix_config: |
experimental-features = flakes nix-command
- run: echo NIX_VERSION="$(nix eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
- run: nix build .#dockerImage -L
install_url: https://releases.nixos.org/nix/nix-2.20.3/install
- uses: DeterminateSystems/magic-nix-cache-action@main
- run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV
- run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L
- run: docker load -i ./result/image.tar.gz
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION
- run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master
@@ -293,6 +289,7 @@ jobs:
extra_nix_config:
experimental-features = nix-command flakes
github_token: ${{ secrets.GITHUB_TOKEN }}
- uses: DeterminateSystems/magic-nix-cache-action@main
- run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh
profile_build:
@@ -313,6 +310,7 @@ jobs:
extra_nix_config: |
experimental-features = flakes nix-command ca-derivations impure-derivations
max-jobs = 1
- uses: DeterminateSystems/magic-nix-cache-action@main
- run: |
nix build -L --file ./ci/gha/profile-build buildTimeReport --out-link build-time-report.md
cat build-time-report.md >> $GITHUB_STEP_SUMMARY
-35
View File
@@ -1,35 +0,0 @@
name: "Coverity Scan"
on:
# FIXME: This is here for testing, should be removed before merging
pull_request:
push:
branches:
- master
- coverity-test
permissions: read-all
jobs:
coverity:
name: Coverity static analysis
runs-on: ubuntu-24.04
timeout-minutes: 120
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- uses: ./.github/actions/install-nix-action
with:
dogfood: false
extra_nix_config: experimental-features = nix-command flakes
- name: configure
run: nix develop --configure
- uses: vapier/coverity-scan-action@v1
with:
project: NixOS/nix
email: ${{ secrets.COVERITY_SCAN_EMAIL }}
token: ${{ secrets.COVERITY_SCAN_TOKEN }}
build_language: cxx
command: nix develop --build
version: ${{ github.sha }}
+1 -3
View File
@@ -34,10 +34,8 @@ mkMesonDerivation (finalAttrs: {
(fileset.unions [
../../.version
# For example JSON
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path
../../src/libstore-tests/data/derived-path
# Too many different types of files to filter for now
../../doc/manual
./.
@@ -1,14 +0,0 @@
---
synopsis: "S3 URLs now support object versioning via versionId parameter"
prs: [14274]
issues: [13955]
---
S3 URLs now support a `versionId` query parameter to fetch specific versions
of objects from S3 buckets with versioning enabled. This allows pinning to
exact object versions for reproducibility and protection against unexpected
changes:
```
s3://bucket/key?region=us-east-1&versionId=abc123def456
```
+1 -3
View File
@@ -117,12 +117,10 @@
- [Architecture and Design](architecture/architecture.md)
- [Formats and Protocols](protocols/index.md)
- [JSON Formats](protocols/json/index.md)
- [File System Object](protocols/json/file-system-object.md)
- [Hash](protocols/json/hash.md)
- [Content Address](protocols/json/content-address.md)
- [Store Path](protocols/json/store-path.md)
- [Store Object Info](protocols/json/store-object-info.md)
- [Derivation](protocols/json/derivation.md)
- [Deriving Path](protocols/json/deriving-path.md)
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
- [Store Path Specification](protocols/store-path.md)
- [Nix Archive (NAR) Format](protocols/nix-archive.md)
@@ -48,7 +48,8 @@ The behaviour of the collector is also influenced by the
configuration file.
By default, the collector prints the total number of freed bytes when it
finishes (or when it is interrupted).
finishes (or when it is interrupted). With `--print-dead`, it prints the
number of bytes that would be freed.
{{#include ./opt-common.md}}
@@ -1,21 +0,0 @@
{{#include content-address-v1-fixed.md}}
## Examples
### [Text](@docroot@/store/store-object/content-address.html#method-text) method
```json
{{#include schema/content-address-v1/text.json}}
```
### [Nix Archive](@docroot@/store/store-object/content-address.html#method-nix-archive) method
```json
{{#include schema/content-address-v1/nar.json}}
```
<!-- need to convert YAML to JSON first
## Raw Schema
[JSON Schema for Hash v1](schema/content-address-v1.json)
-->
@@ -1,21 +0,0 @@
{{#include deriving-path-v1-fixed.md}}
## Examples
### Constant
```json
{{#include schema/deriving-path-v1/single_opaque.json}}
```
### Output of static derivation
```json
{{#include schema/deriving-path-v1/single_built.json}}
```
### Output of dynamic derivation
```json
{{#include schema/deriving-path-v1/single_built_built.json}}
```
@@ -0,0 +1,21 @@
{{#include file-system-object-v1-fixed.md}}
## Examples
### Simple
```json
{{#include schema/file-system-object-v1/simple.json}}
```
### Complex
```json
{{#include schema/file-system-object-v1/complex.json}}
```
<!--
## Raw Schema
[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
-->
@@ -12,6 +12,4 @@ s/\\`/`/g
# As we have more such relative links, more replacements of this nature
# should appear below.
s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
s^\(./content-address-v1.yaml\)^[JSON format for `ContentAddress`](./content-address.html)^g
s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
+1 -3
View File
@@ -9,11 +9,9 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
json_schema_config = files('json-schema-for-humans-config.yaml')
schemas = [
'file-system-object-v1',
'hash-v1',
'content-address-v1',
'store-path-v1',
'derivation-v3',
'deriving-path-v1',
]
schema_files = files()
@@ -1 +0,0 @@
../../../../../../src/libstore-tests/data/content-address
@@ -1,55 +0,0 @@
"$schema": "http://json-schema.org/draft-04/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/content-address-v1.json"
title: Content Address
description: |
This schema describes the JSON representation of Nix's `ContentAddress` type, which conveys information about [content-addressing store objects](@docroot@/store/store-object/content-address.md).
> **Note**
>
> For current methods of content addressing, this data type is a bit suspicious, because it is neither simply a content address of a file system object (the `method` is richer), nor simply a content address of a store object (the `hash` doesn't account for the references).
> It should thus only be used in contexts where the references are also known / otherwise made tamper-resistant.
<!--
TODO currently `ContentAddress` is used in both of these, and so same rationale applies, but actually in both cases the JSON is currently ad-hoc.
That will be fixed, and as each is fixed, the example (along with a more precise link to the field in question) should be become part of the above note, so what is is saying is more clear.
> For example:
> - Fixed outputs of derivations are not allowed to have any references, so an empty reference set is statically known by assumption.
> - [Store object info](./store-object-info.md) includes the set of references along side the (optional) content address.
> This data type is thus safely used in both of these contexts.
-->
type: object
properties:
method:
"$ref": "#/$defs/method"
hash:
title: Content Address
description: |
This would be the content-address itself.
For all current methods, this is just a content address of the file system object of the store object, [as described in the store chapter](@docroot@/store/file-system-object/content-address.md), and not of the store object as a whole.
In particular, the references of the store object are *not* taken into account with this hash (and currently-supported methods).
"$ref": "./hash-v1.yaml"
required:
- method
- hash
additionalProperties: false
"$defs":
method:
type: string
enum: [flat, nar, text, git]
title: Content-Addressing Method
description: |
A string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
Valid method strings are:
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat) (provided the contents are a single file)
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
@@ -1,5 +1,5 @@
"$schema": "http://json-schema.org/draft-04/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json"
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/derivation-v3.json
title: Derivation
description: |
Experimental JSON representation of a Nix derivation (version 3).
@@ -85,7 +85,7 @@ properties:
> ]
> ```
items:
$ref: "store-path-v1.yaml"
type: string
inputDrvs:
type: object
@@ -103,15 +103,6 @@ properties:
> ```
>
> specifies that this derivation depends on the `dev` output of `curl`, and the `out` output of `unzip`.
patternProperties:
"^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\\.drv$":
title: Store Path
description: |
A store path to a derivation, mapped to the outputs of that derivation.
oneOf:
- "$ref": "#/$defs/outputNames"
- "$ref": "#/$defs/dynamicOutputs"
additionalProperties: false
system:
type: string
@@ -157,16 +148,25 @@ properties:
type: object
properties:
path:
$ref: "store-path-v1.yaml"
type: string
title: Output path
description: |
The output path, if known in advance.
method:
"$ref": "./content-address-v1.yaml#/$defs/method"
type: string
title: Content addressing method
enum: [flat, nar, text, git]
description: |
For an output which will be [content addressed](@docroot@/store/derivation/outputs/content-address.md), a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen.
See the linked original definition for further details.
Valid method strings are:
- [`flat`](@docroot@/store/store-object/content-address.md#method-flat)
- [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive)
- [`text`](@docroot@/store/store-object/content-address.md#method-text)
- [`git`](@docroot@/store/store-object/content-address.md#method-git)
hashAlgo:
title: Hash algorithm
"$ref": "./hash-v1.yaml#/$defs/algorithm"
@@ -176,28 +176,3 @@ properties:
title: Expected hash value
description: |
For fixed-output derivations, the expected content hash in base-16.
outputName:
type: string
title: Output name
description: Name of the derivation output to depend on
outputNames:
type: array
title: Output Names
description: Set of names of derivation outputs to depend on
items:
"$ref": "#/$defs/outputName"
dynamicOutputs:
type: object
title: Dynamic Outputs
description: |
**Experimental feature**: [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations)
This recursive data type allows for depending on outputs of outputs.
properties:
outputs:
"$ref": "#/$defs/outputNames"
dynamicOutputs:
"$ref": "#/$defs/dynamicOutputs"
@@ -1 +0,0 @@
../../../../../../src/libstore-tests/data/derived-path
@@ -1,27 +0,0 @@
"$schema": "http://json-schema.org/draft-04/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/deriving-path-v1.json"
title: Deriving Path
description: |
This schema describes the JSON representation of Nix's [Deriving Path](@docroot@/store/derivation/index.md#deriving-path).
oneOf:
- title: Constant
description: |
See [Constant](@docroot@/store/derivation/index.md#deriving-path-constant) deriving path.
$ref: "store-path-v1.yaml"
- title: Output
description: |
See [Output](@docroot@/store/derivation/index.md#deriving-path-output) deriving path.
type: object
properties:
drvPath:
"$ref": "#"
description: |
A deriving path to a [Derivation](@docroot@/store/derivation/index.md#store-derivation), whose output is being referred to.
output:
type: string
description: |
The name of an output produced by that derivation (e.g. "out", "doc", etc.).
required:
- drvPath
- output
additionalProperties: false
@@ -0,0 +1 @@
../../../../../../src/libutil-tests/data/memory-source-accessor
@@ -0,0 +1,65 @@
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
title: File System Object
description: |
This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
The schema is recursive because file system objects contain other file system objects.
type: object
required: ["type"]
properties:
type:
type: string
enum: ["regular", "symlink", "directory"]
# Enforce conditional structure based on `type`
anyOf:
- $ref: "#/$defs/regular"
required: ["type", "contents"]
- $ref: "#/$defs/symlink"
required: ["type", "target"]
- $ref: "#/$defs/directory"
required: ["type", "contents"]
"$defs":
regular:
title: Regular File
required: ["contents"]
properties:
type:
const: "regular"
contents:
type: string
description: Base64-encoded file contents
executable:
type: boolean
description: Whether the file is executable.
default: false
additionalProperties: false
symlink:
title: Symbolic Link
required: ["target"]
properties:
type:
const: "symlink"
target:
type: string
description: Target path of the symlink.
additionalProperties: false
directory:
title: Directory
required: ["contents"]
properties:
type:
const: "directory"
contents:
type: object
description: |
Map of names to nested file system objects (for type=directory)
additionalProperties:
$ref: "#"
additionalProperties: false
@@ -1,5 +1,5 @@
"$schema": "http://json-schema.org/draft-04/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json"
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/hash-v1.json
title: Hash
description: |
A cryptographic hash value used throughout Nix for content addressing and integrity verification.
@@ -1 +0,0 @@
../../../../../../src/libstore-tests/data/store-path
@@ -1,32 +0,0 @@
"$schema": "http://json-schema.org/draft-07/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/store-path-v1.json"
title: Store Path
description: |
A [store path](@docroot@/store/store-path.md) identifying a store object.
This schema describes the JSON representation of store paths as used in various Nix JSON APIs.
> **Warning**
>
> This JSON format is currently
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command)
> and subject to change.
## Format
Store paths in JSON are represented as strings containing just the hash and name portion, without the store directory prefix.
For example: `"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"`
(If the store dir is `/nix/store`, then this corresponds to the path `/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv`.)
## Structure
The format follows this pattern: `${digest}-${name}`
- **hash**: Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters)
- **name**: The package name and optional version/suffix information
type: string
pattern: "^[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+$"
minLength: 34
@@ -1,15 +0,0 @@
{{#include store-path-v1-fixed.md}}
## Examples
### Simple store path
```json
{{#include schema/store-path-v1/simple.json}}
```
<!--
## Raw Schema
[JSON Schema for Store Path v1](schema/store-path-v1.json)
-->
-21
View File
@@ -471,27 +471,6 @@
}
);
apps = forAllSystems (
system:
let
pkgs = nixpkgsFor.${system}.native;
opener = if pkgs.stdenv.isDarwin then "open" else "xdg-open";
in
{
open-manual = {
type = "app";
program = "${pkgs.writeShellScript "open-nix-manual" ''
manual_path="${self.packages.${system}.nix-manual}/share/doc/nix/manual/index.html"
if ! ${opener} "$manual_path"; then
echo "Failed to open manual with ${opener}. Manual is located at:"
echo "$manual_path"
fi
''}";
meta.description = "Open the Nix manual in your browser";
};
}
);
devShells =
let
makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; };
+1 -1
View File
@@ -1,5 +1,5 @@
#compdef nix
# shellcheck disable=all
#compdef nix
function _nix() {
local ifs_bk="$IFS"
-1
View File
@@ -1 +0,0 @@
../../src/libstore-tests/data/content-address
-1
View File
@@ -1 +0,0 @@
../../src/libstore-tests/data/derived-path
+1
View File
@@ -0,0 +1 @@
../../src/libutil-tests/data/memory-source-accessor
+22 -37
View File
@@ -20,6 +20,14 @@ schema_dir = meson.current_source_dir() / 'schema'
# Get all example files
schemas = [
{
'stem' : 'file-system-object',
'schema' : schema_dir / 'file-system-object-v1.yaml',
'files' : [
'simple.json',
'complex.json',
],
},
{
'stem' : 'hash',
'schema' : schema_dir / 'hash-v1.yaml',
@@ -30,21 +38,6 @@ schemas = [
'blake3-base64.json',
],
},
{
'stem' : 'content-address',
'schema' : schema_dir / 'content-address-v1.yaml',
'files' : [
'text.json',
'nar.json',
],
},
{
'stem' : 'store-path',
'schema' : schema_dir / 'store-path-v1.yaml',
'files' : [
'simple.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml',
@@ -53,28 +46,20 @@ schemas = [
'simple-derivation.json',
],
},
{
'stem' : 'derivation',
'schema' : schema_dir / 'derivation-v3.yaml#/$defs/output',
'files' : [
'output-caFixedFlat.json',
'output-caFixedNAR.json',
'output-caFixedText.json',
'output-caFloating.json',
'output-deferred.json',
'output-impure.json',
'output-inputAddressed.json',
],
},
{
'stem' : 'deriving-path',
'schema' : schema_dir / 'deriving-path-v1.yaml',
'files' : [
'single_opaque.json',
'single_built.json',
'single_built_built.json',
],
},
# # Not sure how to make subschema work
# {
# 'stem': 'derivation',
# 'schema': schema_dir / 'derivation-v3.yaml#output',
# 'files' : [
# 'output-caFixedFlat.json',
# 'output-caFixedNAR.json',
# 'output-caFixedText.json',
# 'output-caFloating.json',
# 'output-deferred.json',
# 'output-impure.json',
# 'output-inputAddressed.json',
# ],
# },
]
# Validate each example against the schema
+1 -3
View File
@@ -20,11 +20,9 @@ mkMesonDerivation (finalAttrs: {
fileset = lib.fileset.unions [
../../.version
../../doc/manual/source/protocols/json/schema
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path
../../src/libstore-tests/data/derivation
../../src/libstore-tests/data/derived-path
./.
];
-1
View File
@@ -1 +0,0 @@
../../src/libstore-tests/data/store-path
+3 -4
View File
@@ -10,7 +10,7 @@ using namespace testing;
struct ValuePrintingTests : LibExprTest
{
template<class... A>
void test(Value & v, std::string_view expected, A... args)
void test(Value v, std::string_view expected, A... args)
{
std::stringstream out;
v.print(state, out, args...);
@@ -625,11 +625,10 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided)
vThree.mkInt(3);
builder.insert(state.symbols.create("three"), &vThree);
Value vAttrs2;
vAttrs2.mkAttrs(builder.finish());
vAttrs.mkAttrs(builder.finish());
test(
vAttrs2,
vAttrs,
"{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }",
PrintOptions{.ansiColors = true, .maxAttrs = 1});
}
+7 -11
View File
@@ -2050,10 +2050,10 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
};
// List of returned strings. References to these Values must NOT be persisted.
SmallTemporaryValueVector<conservativeStackReservation> values(es.size());
SmallTemporaryValueVector<conservativeStackReservation> values(es->size());
Value * vTmpP = values.data();
for (auto & [i_pos, i] : es) {
for (auto & [i_pos, i] : *es) {
Value & vTmp = *vTmpP++;
i->eval(state, env, vTmp);
@@ -2097,7 +2097,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
.debugThrow();
} else {
if (s.empty())
s.reserve(es.size());
s.reserve(es->size());
/* skip canonization of first path, which would only be not
canonized in the first place if it's coming from a ./${foo} type
path */
@@ -3067,7 +3067,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path)
return parseExprFromFile(path, staticBaseEnv);
}
Expr * EvalState::parseExprFromFile(const SourcePath & path, const std::shared_ptr<StaticEnv> & staticEnv)
Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv)
{
auto buffer = path.resolveSymlinks().readFile();
// readFile hopefully have left some extra space for terminators
@@ -3075,8 +3075,8 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, const std::shared_p
return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv);
}
Expr * EvalState::parseExprFromString(
std::string s_, const SourcePath & basePath, const std::shared_ptr<StaticEnv> & staticEnv)
Expr *
EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
{
// NOTE this method (and parseStdin) must take care to *fully copy* their input
// into their respective Pos::Origin until the parser stops overwriting its input
@@ -3210,11 +3210,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
}
Expr * EvalState::parse(
char * text,
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
const std::shared_ptr<StaticEnv> & staticEnv)
char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv)
{
DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath
DocCommentMap * docComments = &tmpDocComments;
+4 -41
View File
@@ -5,7 +5,6 @@
#include "nix/expr/symbol-table.hh"
#include <boost/container/static_vector.hpp>
#include <boost/iterator/function_output_iterator.hpp>
#include <algorithm>
#include <functional>
@@ -464,48 +463,12 @@ private:
return bindings->baseLayer;
}
/**
* If the bindings gets "layered" on top of another we need to recalculate
* the number of unique attributes in the chain.
*
* This is done by either iterating over the base "layer" and the newly added
* attributes and counting duplicates. If the base "layer" is big this approach
* is inefficient and we fall back to doing per-element binary search in the base
* "layer".
*/
void finishSizeIfNecessary()
{
if (!hasBaseLayer())
return;
auto & base = *bindings->baseLayer;
auto attrs = std::span(bindings->attrs, bindings->numAttrs);
Bindings::size_type duplicates = 0;
/* If the base bindings is smaller than the newly added attributes
iterate using std::set_intersection to run in O(|base| + |attrs|) =
O(|attrs|). Otherwise use an O(|attrs| * log(|base|)) per-attr binary
search to check for duplicates. Note that if we are in this code path then
|attrs| <= bindingsUpdateLayerRhsSizeThreshold, which 16 by default. We are
optimizing for the case when a small attribute set gets "layered" on top of
a much larger one. When attrsets are already small it's fine to do a linear
scan, but we should avoid expensive iterations over large "base" attrsets. */
if (attrs.size() > base.size()) {
std::set_intersection(
base.begin(),
base.end(),
attrs.begin(),
attrs.end(),
boost::make_function_output_iterator([&]([[maybe_unused]] auto && _) { ++duplicates; }));
} else {
for (const auto & attr : attrs) {
if (base.get(attr.name))
++duplicates;
}
}
bindings->numAttrsInChain = base.numAttrsInChain + attrs.size() - duplicates;
if (hasBaseLayer())
/* NOTE: Do not use std::ranges::distance, since Bindings is a sized
range, but we are calculating this size here. */
bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end());
}
public:
+11 -13
View File
@@ -191,7 +191,7 @@ std::ostream & operator<<(std::ostream & os, const ValueType t);
struct RegexCache;
ref<RegexCache> makeRegexCache();
std::shared_ptr<RegexCache> makeRegexCache();
struct DebugTrace
{
@@ -372,7 +372,6 @@ public:
const fetchers::Settings & fetchSettings;
const EvalSettings & settings;
SymbolTable symbols;
PosTable positions;
@@ -419,7 +418,7 @@ public:
RootValue vImportedDrvToDerivation = nullptr;
const ref<fetchers::InputCache> inputCache;
ref<fetchers::InputCache> inputCache;
/**
* Debugger
@@ -472,18 +471,18 @@ private:
/* Cache for calls to addToStore(); maps source paths to the store
paths. */
const ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
/**
* A cache that maps paths to "resolved" paths for importing Nix
* expressions, i.e. `/foo` to `/foo/default.nix`.
*/
const ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
ref<boost::concurrent_flat_map<SourcePath, SourcePath>> importResolutionCache;
/**
* A cache from resolved paths to values.
*/
const ref<boost::concurrent_flat_map<
ref<boost::concurrent_flat_map<
SourcePath,
Value *,
std::hash<SourcePath>,
@@ -505,7 +504,7 @@ private:
/**
* Cache used by prim_match().
*/
const ref<RegexCache> regexCache;
std::shared_ptr<RegexCache> regexCache;
public:
@@ -593,13 +592,12 @@ public:
* Parse a Nix expression from the specified file.
*/
Expr * parseExprFromFile(const SourcePath & path);
Expr * parseExprFromFile(const SourcePath & path, const std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);
/**
* Parse a Nix expression from the specified string.
*/
Expr *
parseExprFromString(std::string s, const SourcePath & basePath, const std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv);
Expr * parseExprFromString(std::string s, const SourcePath & basePath);
Expr * parseStdin();
@@ -768,7 +766,7 @@ public:
#if NIX_USE_BOEHMGC
/** A GC root for the baseEnv reference. */
const std::shared_ptr<Env *> baseEnvP;
std::shared_ptr<Env *> baseEnvP;
#endif
public:
@@ -782,7 +780,7 @@ public:
/**
* The same, but used during parsing to resolve variables.
*/
const std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private
/**
* Internal primops not exposed to the user.
@@ -864,7 +862,7 @@ private:
size_t length,
Pos::Origin origin,
const SourcePath & basePath,
const std::shared_ptr<StaticEnv> & staticEnv);
std::shared_ptr<StaticEnv> & staticEnv);
/**
* Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run
+5 -11
View File
@@ -442,14 +442,8 @@ struct ExprAttrs : Expr
struct ExprList : Expr
{
std::span<Expr *> elems;
ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::vector<Expr *> exprs)
: elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()})
{
std::ranges::copy(exprs, elems.begin());
};
std::vector<Expr *> elems;
ExprList() {};
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env) override;
@@ -701,11 +695,11 @@ struct ExprConcatStrings : Expr
{
PosIdx pos;
bool forceString;
std::vector<std::pair<PosIdx, Expr *>> es;
ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> && es)
std::vector<std::pair<PosIdx, Expr *>> * es;
ExprConcatStrings(const PosIdx & pos, bool forceString, std::vector<std::pair<PosIdx, Expr *>> * es)
: pos(pos)
, forceString(forceString)
, es(std::move(es)) {};
, es(es) {};
PosIdx getPos() const override
{
+9 -7
View File
@@ -282,7 +282,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
}
/* Strip spaces from each line. */
std::vector<std::pair<PosIdx, Expr *>> es2{};
auto * es2 = new std::vector<std::pair<PosIdx, Expr *>>;
atStartOfLine = true;
size_t curDropped = 0;
size_t n = es.size();
@@ -290,7 +290,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
const auto trimExpr = [&](Expr * e) {
atStartOfLine = false;
curDropped = 0;
es2.emplace_back(i->first, e);
es2->emplace_back(i->first, e);
};
const auto trimString = [&](const StringToken & t) {
std::string s2;
@@ -324,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
// Ignore empty strings for a minor optimisation and AST simplification
if (s2 != "") {
es2.emplace_back(i->first, new ExprString(alloc, s2));
es2->emplace_back(i->first, new ExprString(alloc, s2));
}
};
for (; i != es.end(); ++i, --n) {
@@ -333,17 +333,19 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
// If there is nothing at all, return the empty string directly.
// This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
if (es2.size() == 0) {
if (es2->size() == 0) {
auto * const result = new ExprString("");
delete es2;
return result;
}
/* If this is a single string, then don't do a concatenation. */
if (es2.size() == 1 && dynamic_cast<ExprString *>((es2)[0].second)) {
auto * const result = (es2)[0].second;
if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) {
auto * const result = (*es2)[0].second;
delete es2;
return result;
}
return new ExprConcatStrings(pos, true, std::move(es2));
return new ExprConcatStrings(pos, true, es2);
}
inline PosIdx LexerState::at(const ParserLocation & loc)
+17 -17
View File
@@ -142,11 +142,11 @@ or { return OR_KW; }
return PIPE_INTO;
}
{ID} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return ID; }
{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; }
{INT} { errno = 0;
std::optional<int64_t> numMay = string2Int<int64_t>(yytext);
if (numMay.has_value()) {
yylval->emplace<NixInt>(*numMay);
yylval->n = NixInt{*numMay};
} else {
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid integer '%1%'", yytext),
@@ -156,7 +156,7 @@ or { return OR_KW; }
return INT_LIT;
}
{FLOAT} { errno = 0;
yylval->emplace<NixFloat>(strtod(yytext, 0));
yylval->nf = strtod(yytext, 0);
if (errno != 0)
throw ParseError(ErrorInfo{
.msg = HintFmt("invalid float '%1%'", yytext),
@@ -183,7 +183,7 @@ or { return OR_KW; }
/* It is impossible to match strings ending with '$' with one
regex because trailing contexts are only valid at the end
of a rule. (A sane but undocumented limitation.) */
yylval->emplace<StringToken>(unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; }));
yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; });
return STR;
}
<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
@@ -198,27 +198,27 @@ or { return OR_KW; }
\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; }
<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ {
yylval->emplace<StringToken>(yytext, (size_t) yyleng, true);
forceNoNullByte(yylval->as<StringToken>(), [&]() { return state->positions[CUR_POS]; });
yylval->str = {yytext, (size_t) yyleng, true};
forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; });
return IND_STR;
}
<IND_STRING>\'\'\$ |
<IND_STRING>\$ {
yylval->emplace<StringToken>("$", 1);
yylval->str = {"$", 1};
return IND_STR;
}
<IND_STRING>\'\'\' {
yylval->emplace<StringToken>("''", 2);
yylval->str = {"''", 2};
return IND_STR;
}
<IND_STRING>\'\'\\{ANY} {
yylval->emplace<StringToken>(unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; }));
yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; });
return IND_STR;
}
<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; }
<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; }
<IND_STRING>\' {
yylval->emplace<StringToken>("'", 1);
yylval->str = {"'", 1};
return IND_STR;
}
@@ -232,14 +232,14 @@ or { return OR_KW; }
<PATH_START>{PATH_SEG} {
POP_STATE();
PUSH_STATE(INPATH_SLASH);
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
yylval->path = {yytext, (size_t) yyleng};
return PATH;
}
<PATH_START>{HPATH_START} {
POP_STATE();
PUSH_STATE(INPATH_SLASH);
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
yylval->path = {yytext, (size_t) yyleng};
return HPATH;
}
@@ -248,7 +248,7 @@ or { return OR_KW; }
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
yylval->path = {yytext, (size_t) yyleng};
return PATH;
}
{HPATH} {
@@ -256,7 +256,7 @@ or { return OR_KW; }
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
yylval->path = {yytext, (size_t) yyleng};
return HPATH;
}
@@ -272,7 +272,7 @@ or { return OR_KW; }
PUSH_STATE(INPATH_SLASH);
else
PUSH_STATE(INPATH);
yylval->emplace<StringToken>(yytext, (size_t) yyleng);
yylval->str = {yytext, (size_t) yyleng};
return STR;
}
<INPATH>{ANY} |
@@ -294,8 +294,8 @@ or { return OR_KW; }
});
}
{SPATH} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return SPATH; }
{URI} { yylval->emplace<StringToken>(yytext, (size_t) yyleng); return URI; }
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; }
%{
// Doc comment rule
+2 -47
View File
@@ -183,62 +183,17 @@ subdir('primops')
subdir('nix-meson-build-support/export-all-symbols')
subdir('nix-meson-build-support/windows-version')
# Turns out that Bison/Flex are particularly sensitive to compilers
# failing to inline functions. For that reason we crank up the inlining
# threshold manually for optimized builds. Yes, this can be considered 'ricing'
# the compiler, but it does pay off.
#
# NOTE: missed inlining can be spotted (for Clang) using -Rpass-missed=inline
# and -fdump-ipa-inline-missed (for GCC).
parser_library_cpp_args = []
if not get_option('debug')
if cxx.get_id() == 'clang'
# The default as of LLVM 21 is 225:
# llc --help-hidden | grep inline-threshold
parser_library_cpp_args += [
'-mllvm',
'-inline-threshold=5000',
]
elif cxx.get_id() == 'gcc'
parser_library_cpp_args += [
'--param=max-inline-insns-single=1000',
'--param=max-inline-insns-auto=1000',
'--param=inline-unit-growth=400',
]
endif
endif
# Working around https://github.com/mesonbuild/meson/issues/1367.
parser_library = static_library(
'nixexpr-parser',
parser_tab,
lexer_tab,
cpp_args : parser_library_cpp_args,
dependencies : deps_public + deps_private + deps_other,
include_directories : include_dirs,
# 1. Stdlib and regular assertions regress parser performance significantly, so build without
# them for this one library when building in a release configuration.
# 2. Disable LTO for GCC because then inlining flags won't apply, since LTO in GCC is done
# by plonking down GIMPLE in the archive.
override_options : [
'b_ndebug=@0@'.format(not get_option('debug')),
'b_lto=@0@'.format(get_option('b_lto') and cxx.get_id() != 'gcc'),
],
)
this_library = library(
'nixexpr',
sources,
config_priv_h,
parser_tab[1],
lexer_tab[1],
parser_tab,
lexer_tab,
generated_headers,
soversion : nix_soversion,
dependencies : deps_public + deps_private + deps_other,
include_directories : include_dirs,
link_args : linker_export_flags,
link_whole : [ parser_library ],
prelink : true, # For C++ static initializers
install : true,
cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [],
+2 -2
View File
@@ -246,7 +246,7 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co
{
bool first = true;
str << "(";
for (auto & i : es) {
for (auto & i : *es) {
if (first)
first = false;
else
@@ -564,7 +564,7 @@ void ExprConcatStrings::bindVars(EvalState & es, const std::shared_ptr<const Sta
if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env));
for (auto & i : this->es)
for (auto & i : *this->es)
i.second->bindVars(es, env);
}
+98 -74
View File
@@ -14,10 +14,6 @@
%code requires {
// bison adds a bunch of switch statements with default:
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
#ifndef BISON_HEADER
#define BISON_HEADER
@@ -124,28 +120,46 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
%}
%define api.value.type variant
%union {
// !!! We're probably leaking stuff here.
nix::Expr * e;
nix::ExprList * list;
nix::ExprAttrs * attrs;
nix::Formals * formals;
nix::Formal * formal;
nix::NixInt n;
nix::NixFloat nf;
nix::StringToken id; // !!! -> Symbol
nix::StringToken path;
nix::StringToken uri;
nix::StringToken str;
std::vector<nix::AttrName> * attrNames;
std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
std::variant<nix::Expr *, std::string_view> * to_be_string;
std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
}
%type <nix::Expr *> start expr expr_function expr_if expr_op
%type <nix::Expr *> expr_select expr_simple expr_app
%type <nix::Expr *> expr_pipe_from expr_pipe_into
%type <std::vector<Expr *>> list
%type <nix::ExprAttrs *> binds binds1
%type <nix::Formals *> formals formal_set
%type <nix::Formal> formal
%type <std::vector<nix::AttrName>> attrpath
%type <std::vector<std::pair<nix::AttrName, nix::PosIdx>>> attrs
%type <std::vector<std::pair<nix::PosIdx, nix::Expr *>>> string_parts_interpolated
%type <std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>>> ind_string_parts
%type <nix::Expr *> path_start
%type <std::variant<nix::Expr *, std::string_view>> string_parts string_attr
%type <nix::StringToken> attr
%token <nix::StringToken> ID
%token <nix::StringToken> STR IND_STR
%token <nix::NixInt> INT_LIT
%token <nix::NixFloat> FLOAT_LIT
%token <nix::StringToken> PATH HPATH SPATH PATH_END
%token <nix::StringToken> URI
%type <e> start expr expr_function expr_if expr_op
%type <e> expr_select expr_simple expr_app
%type <e> expr_pipe_from expr_pipe_into
%type <list> expr_list
%type <attrs> binds binds1
%type <formals> formals formal_set
%type <formal> formal
%type <attrNames> attrpath
%type <inheritAttrs> attrs
%type <string_parts> string_parts_interpolated
%type <ind_string_parts> ind_string_parts
%type <e> path_start
%type <to_be_string> string_parts string_attr
%type <id> attr
%token <id> ID
%token <str> STR IND_STR
%token <n> INT_LIT
%token <nf> FLOAT_LIT
%token <path> PATH HPATH SPATH PATH_END
%token <uri> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */
@@ -247,9 +261,9 @@ expr_op
| expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); }
| expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); }
| expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); }
| expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move($3)); }
| expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move(*$3)); delete $3; }
| expr_op '+' expr_op
{ $$ = new ExprConcatStrings(state->at(@2), false, {{state->at(@1), $1}, {state->at(@3), $3}}); }
{ $$ = new ExprConcatStrings(state->at(@2), false, new std::vector<std::pair<PosIdx, Expr *> >({{state->at(@1), $1}, {state->at(@3), $3}})); }
| expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); }
| expr_op '*' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.mul), {$1, $3}); }
| expr_op '/' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.div), {$1, $3}); }
@@ -268,9 +282,9 @@ expr_app
expr_select
: expr_simple '.' attrpath
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move($3), nullptr); }
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), nullptr); delete $3; }
| expr_simple '.' attrpath OR_KW expr_select
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move($3), $5); $5->warnIfCursedOr(state->symbols, state->positions); }
{ $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); }
| /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
allow stuff like ‘map or [...]’. This production is problematic (see
https://github.com/NixOS/nix/issues/11118) and will be refactored in the
@@ -297,15 +311,17 @@ expr_simple
std::visit(overloaded{
[&](std::string_view str) { $$ = new ExprString(state->alloc, str); },
[&](Expr * expr) { $$ = expr; }},
$2);
*$2);
delete $2;
}
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = state->stripIndentation(CUR_POS, std::move($2));
$$ = state->stripIndentation(CUR_POS, std::move(*$2));
delete $2;
}
| path_start PATH_END
| path_start string_parts_interpolated PATH_END {
$2.insert($2.begin(), {state->at(@1), $1});
$$ = new ExprConcatStrings(CUR_POS, false, std::move($2));
$2->insert($2->begin(), {state->at(@1), $1});
$$ = new ExprConcatStrings(CUR_POS, false, $2);
}
| SPATH {
std::string_view path($1.p + 1, $1.l - 2);
@@ -334,23 +350,24 @@ expr_simple
{ $2->pos = CUR_POS; $$ = $2; }
| '{' '}'
{ $$ = new ExprAttrs(CUR_POS); }
| '[' list ']' { $$ = new ExprList(state->alloc, std::move($2)); }
| '[' expr_list ']' { $$ = $2; }
;
string_parts
: STR { $$ = $1; }
| string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, std::move($1)); }
| { $$ = std::string_view(); }
: STR { $$ = new std::variant<Expr *, std::string_view>($1); }
| string_parts_interpolated { $$ = new std::variant<Expr *, std::string_view>(new ExprConcatStrings(CUR_POS, true, $1)); }
| { $$ = new std::variant<Expr *, std::string_view>(std::string_view()); }
;
string_parts_interpolated
: string_parts_interpolated STR
{ $$ = std::move($1); $$.emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); }
| DOLLAR_CURLY expr '}' { $$.emplace_back(state->at(@1), $2); }
{ $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); }
| string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
| DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<PosIdx, Expr *>>; $$->emplace_back(state->at(@1), $2); }
| STR DOLLAR_CURLY expr '}' {
$$.emplace_back(state->at(@1), new ExprString(state->alloc, $1));
$$.emplace_back(state->at(@2), $3);
$$ = new std::vector<std::pair<PosIdx, Expr *>>;
$$->emplace_back(state->at(@1), new ExprString(state->alloc, $1));
$$->emplace_back(state->at(@2), $3);
}
;
@@ -391,9 +408,9 @@ path_start
;
ind_string_parts
: ind_string_parts IND_STR { $$ = std::move($1); $$.emplace_back(state->at(@2), $2); }
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = std::move($1); $$.emplace_back(state->at(@2), $3); }
| { }
: ind_string_parts IND_STR { $$ = $1; $1->emplace_back(state->at(@2), $2); }
| ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); }
| { $$ = new std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>; }
;
binds
@@ -404,17 +421,19 @@ binds
binds1
: binds1[accum] attrpath '=' expr ';'
{ $$ = $accum;
state->addAttr($$, std::move($attrpath), @attrpath, $expr, @expr);
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
delete $attrpath;
}
| binds[accum] INHERIT attrs ';'
{ $$ = $accum;
for (auto & [i, iPos] : $attrs) {
for (auto & [i, iPos] : *$attrs) {
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs.emplace(
i.symbol,
ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
}
delete $attrs;
}
| binds[accum] INHERIT '(' expr ')' attrs ';'
{ $$ = $accum;
@@ -422,7 +441,7 @@ binds1
$accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$accum->inheritFromExprs->push_back($expr);
auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : $attrs) {
for (auto & [i, iPos] : *$attrs) {
if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs.emplace(
@@ -432,45 +451,51 @@ binds1
iPos,
ExprAttrs::AttrDef::Kind::InheritedFrom));
}
delete $attrs;
}
| attrpath '=' expr ';'
{ $$ = new ExprAttrs;
state->addAttr($$, std::move($attrpath), @attrpath, $expr, @expr);
state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr);
delete $attrpath;
}
;
attrs
: attrs attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($2), state->at(@2)); }
: attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); }
| attrs string_attr
{ $$ = std::move($1);
{ $$ = $1;
std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str), state->at(@2)); },
[&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); },
[&](Expr * expr) {
throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in inherit"),
.pos = state->positions[state->at(@2)]
});
}
}, $2);
}, *$2);
delete $2;
}
| { }
| { $$ = new std::vector<std::pair<AttrName, PosIdx>>; }
;
attrpath
: attrpath '.' attr { $$ = std::move($1); $$.emplace_back(state->symbols.create($3)); }
: attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); }
| attrpath '.' string_attr
{ $$ = std::move($1);
{ $$ = $1;
std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($3));
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
}, *$3);
delete $3;
}
| attr { $$.emplace_back(state->symbols.create($1)); }
| attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(state->symbols.create($1))); }
| string_attr
{ std::visit(overloaded {
[&](std::string_view str) { $$.emplace_back(state->symbols.create(str)); },
[&](Expr * expr) { $$.emplace_back(expr); }
}, std::move($1));
{ $$ = new std::vector<AttrName>;
std::visit(overloaded {
[&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); },
[&](Expr * expr) { $$->push_back(AttrName(expr)); }
}, *$1);
delete $1;
}
;
@@ -480,13 +505,13 @@ attr
;
string_attr
: '"' string_parts '"' { $$ = std::move($2); }
| DOLLAR_CURLY expr '}' { $$ = $2; }
: '"' string_parts '"' { $$ = $2; }
| DOLLAR_CURLY expr '}' { $$ = new std::variant<Expr *, std::string_view>($2); }
;
list
: list expr_select { $$ = std::move($1); $$.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
| { }
expr_list
: expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); }
| { $$ = new ExprList; }
;
formal_set
@@ -499,14 +524,14 @@ formal_set
formals
: formals[accum] ',' formal
{ $$ = $accum; $$->formals.emplace_back(std::move($formal)); }
{ $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; }
| formal
{ $$ = new Formals; $$->formals.emplace_back(std::move($formal)); }
{ $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; }
;
formal
: ID { $$ = Formal{CUR_POS, state->symbols.create($1), 0}; }
| ID '?' expr { $$ = Formal{CUR_POS, state->symbols.create($1), $3}; }
: ID { $$ = new Formal{CUR_POS, state->symbols.create($1), 0}; }
| ID '?' expr { $$ = new Formal{CUR_POS, state->symbols.create($1), $3}; }
;
%%
@@ -557,4 +582,3 @@ Expr * parseExprFromBuf(
}
#pragma GCC diagnostic pop // end ignored "-Wswitch-enum"
+3 -3
View File
@@ -1374,7 +1374,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
pos,
"while evaluating the `__structuredAttrs` "
"attribute passed to builtins.derivationStrict"))
jsonObject = StructuredAttrs{};
jsonObject = StructuredAttrs{.structuredAttrs = json::object()};
/* Check whether null attributes should be ignored. */
bool ignoreNulls = false;
@@ -4611,9 +4611,9 @@ struct RegexCache
}
};
ref<RegexCache> makeRegexCache()
std::shared_ptr<RegexCache> makeRegexCache()
{
return make_ref<RegexCache>();
return std::make_shared<RegexCache>();
}
void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v)
-6
View File
@@ -64,8 +64,6 @@ static void runFetchClosureWithRewrite(
.pos = state.positions[pos]});
}
state.allowClosure(toPath);
state.mkStorePathString(toPath, v);
}
@@ -93,8 +91,6 @@ static void runFetchClosureWithContentAddressedPath(
.pos = state.positions[pos]});
}
state.allowClosure(fromPath);
state.mkStorePathString(fromPath, v);
}
@@ -119,8 +115,6 @@ static void runFetchClosureWithInputAddressedPath(
.pos = state.positions[pos]});
}
state.allowClosure(fromPath);
state.mkStorePathString(fromPath, v);
}
+2 -10
View File
@@ -5,7 +5,6 @@
#include "nix/util/json-utils.hh"
#include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/url.hh"
#include <nlohmann/json.hpp>
@@ -66,12 +65,6 @@ Input Input::fromURL(const Settings & settings, const ParsedURL & url, bool requ
}
}
// Provide a helpful hint when user tries file+git instead of git+file
auto parsedScheme = parseUrlScheme(url.scheme);
if (parsedScheme.application == "file" && parsedScheme.transport == "git") {
throw Error("input '%s' is unsupported; did you mean 'git+file' instead of 'file+git'?", url);
}
throw Error("input '%s' is unsupported", url);
}
@@ -519,11 +512,10 @@ using namespace nix;
fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json & json)
{
fetchers::PublicKey res = {};
auto & obj = getObject(json);
if (auto * type = optionalValueAt(obj, "type"))
if (auto type = optionalValueAt(json, "type"))
res.type = getString(*type);
res.key = getString(valueAt(obj, "key"));
res.key = getString(valueAt(json, "key"));
return res;
}
+1 -1
View File
@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
auto url = api.endpoint + "/objects/batch";
const auto & authHeader = api.authHeader;
FileTransferRequest request(parseURL(url));
request.method = HttpMethod::POST;
request.post = true;
Headers headers;
if (authHeader.has_value())
headers.push_back({"Authorization", *authHeader});
+2 -3
View File
@@ -9,7 +9,6 @@
#include "nix/util/users.hh"
#include "nix/util/fs-sink.hh"
#include "nix/util/sync.hh"
#include "nix/util/util.hh"
#include <git2/attr.h>
#include <git2/blob.h>
@@ -531,12 +530,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
auto act = (Activity *) payload;
act->result(
resFetchStatus,
fmt("%d/%d objects received, %d/%d deltas indexed, %s",
fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
stats->received_objects,
stats->total_objects,
stats->indexed_deltas,
stats->total_deltas,
renderSize(stats->received_bytes)));
stats->received_bytes / (1024.0 * 1024.0)));
return getInterrupted() ? -1 : 0;
}
+2
View File
@@ -89,6 +89,8 @@ extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
std::string showBytes(uint64_t bytes);
struct GCResults;
struct PrintFreed
+34 -31
View File
@@ -6,7 +6,6 @@
#include "nix/main/loggers.hh"
#include "nix/main/progress-bar.hh"
#include "nix/util/signals.hh"
#include "nix/util/util.hh"
#include <algorithm>
#include <exception>
@@ -65,19 +64,18 @@ void printMissing(ref<Store> store, const MissingPaths & missing, Verbosity lvl)
}
if (!missing.willSubstitute.empty()) {
const float downloadSizeMiB = missing.downloadSize / (1024.f * 1024.f);
const float narSizeMiB = missing.narSize / (1024.f * 1024.f);
if (missing.willSubstitute.size() == 1) {
printMsg(
lvl,
"this path will be fetched (%s download, %s unpacked):",
renderSize(missing.downloadSize),
renderSize(missing.narSize));
lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", downloadSizeMiB, narSizeMiB);
} else {
printMsg(
lvl,
"these %d paths will be fetched (%s download, %s unpacked):",
"these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):",
missing.willSubstitute.size(),
renderSize(missing.downloadSize),
renderSize(missing.narSize));
downloadSizeMiB,
narSizeMiB);
}
std::vector<const StorePath *> willSubstituteSorted = {};
std::for_each(missing.willSubstitute.begin(), missing.willSubstitute.end(), [&](const StorePath & p) {
@@ -322,29 +320,34 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
std::string error = ANSI_RED "error:" ANSI_NORMAL " ";
try {
try {
fun();
} catch (...) {
/* Subtle: we have to make sure that any `interrupted'
condition is discharged before we reach printMsg()
below, since otherwise it will throw an (uncaught)
exception. */
setInterruptThrown();
throw;
try {
fun();
} catch (...) {
/* Subtle: we have to make sure that any `interrupted'
condition is discharged before we reach printMsg()
below, since otherwise it will throw an (uncaught)
exception. */
setInterruptThrown();
throw;
}
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
return 1;
}
} catch (Exit & e) {
return e.status;
} catch (UsageError & e) {
logError(e.info());
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
logError(e.info());
return e.info().status;
} catch (std::bad_alloc & e) {
printError(error + "out of memory");
return 1;
} catch (std::exception & e) {
printError(error + e.what());
} catch (...) {
/* In case logger also throws just give up. */
return 1;
}
@@ -408,7 +411,7 @@ RunPager::~RunPager()
PrintFreed::~PrintFreed()
{
if (show)
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), renderSize(results.bytesFreed));
std::cout << fmt("%d store paths deleted, %s freed\n", results.paths.size(), showBytes(results.bytesFreed));
}
} // namespace nix
-38
View File
@@ -126,36 +126,6 @@ StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const c
NIXC_CATCH_ERRS_NULL
}
nix_err nix_store_get_fs_closure(
nix_c_context * context,
Store * store,
const StorePath * store_path,
bool flip_direction,
bool include_outputs,
bool include_derivers,
void * userdata,
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path))
{
if (context)
context->last_err_code = NIX_OK;
try {
const auto nixStore = store->ptr;
nix::StorePathSet set;
nixStore->computeFSClosure(store_path->path, set, flip_direction, include_outputs, include_derivers);
if (callback) {
for (const auto & path : set) {
const StorePath tmp{path};
callback(context, userdata, &tmp);
if (context && context->last_err_code != NIX_OK)
return context->last_err_code;
}
}
}
NIXC_CATCH_ERRS
}
nix_err nix_store_realise(
nix_c_context * context,
Store * store,
@@ -173,14 +143,6 @@ nix_err nix_store_realise(
const auto nixStore = store->ptr;
auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore);
assert(results.size() == 1);
// Check if any builds failed
for (auto & result : results) {
if (auto * failureP = result.tryGetFailure())
failureP->rethrow();
}
if (callback) {
for (const auto & result : results) {
if (auto * success = result.tryGetSuccess()) {
-31
View File
@@ -186,8 +186,6 @@ nix_err nix_store_real_path(
* @param[in] path Path to build
* @param[in] userdata data to pass to every callback invocation
* @param[in] callback called for every realised output
* @return NIX_OK if the build succeeded, or an error code if the build/scheduling/outputs/copying/etc failed.
* On error, the callback is never invoked and error information is stored in context.
*/
nix_err nix_store_realise(
nix_c_context * context,
@@ -247,35 +245,6 @@ void nix_derivation_free(nix_derivation * drv);
*/
nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path);
/**
* @brief Gets the closure of a specific store path
*
* @note The callback borrows each StorePath only for the duration of the call.
*
* @param[out] context Optional, stores error information
* @param[in] store nix store reference
* @param[in] store_path The path to compute from
* @param[in] flip_direction If false, compute the forward closure (paths referenced by any store path in the closure).
* If true, compute the backward closure (paths that reference any store path in the closure).
* @param[in] include_outputs If flip_direction is false: for any derivation in the closure, include its outputs.
* If flip_direction is true: for any output in the closure, include derivations that produce
* it.
* @param[in] include_derivers If flip_direction is false: for any output in the closure, include the derivation that
* produced it.
* If flip_direction is true: for any derivation in the closure, include its outputs.
* @param[in] callback The function to call for every store path, in no particular order
* @param[in] userdata The userdata to pass to the callback
*/
nix_err nix_store_get_fs_closure(
nix_c_context * context,
Store * store,
const StorePath * store_path,
bool flip_direction,
bool include_outputs,
bool include_derivers,
void * userdata,
void (*callback)(nix_c_context * context, void * userdata, const StorePath * store_path));
// cffi end
#ifdef __cplusplus
}
+18 -79
View File
@@ -1,7 +1,6 @@
#include <gtest/gtest.h>
#include "nix/store/content-address.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
@@ -9,93 +8,33 @@ namespace nix {
* ContentAddressMethod::parse, ContentAddressMethod::render
* --------------------------------------------------------------------------*/
static auto methods = ::testing::Values(
std::pair{ContentAddressMethod::Raw::Text, "text"},
std::pair{ContentAddressMethod::Raw::Flat, "flat"},
std::pair{ContentAddressMethod::Raw::NixArchive, "nar"},
std::pair{ContentAddressMethod::Raw::Git, "git"});
struct ContentAddressMethodTest : ::testing::Test,
::testing::WithParamInterface<std::pair<ContentAddressMethod, std::string_view>>
{};
TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_1)
TEST(ContentAddressMethod, testRoundTripPrintParse_1)
{
auto & [cam, _] = GetParam();
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
for (ContentAddressMethod cam : {
ContentAddressMethod::Raw::Text,
ContentAddressMethod::Raw::Flat,
ContentAddressMethod::Raw::NixArchive,
ContentAddressMethod::Raw::Git,
}) {
EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam);
}
}
TEST_P(ContentAddressMethodTest, testRoundTripPrintParse_2)
TEST(ContentAddressMethod, testRoundTripPrintParse_2)
{
auto & [cam, camS] = GetParam();
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
for (const std::string_view camS : {
"text",
"flat",
"nar",
"git",
}) {
EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS);
}
}
INSTANTIATE_TEST_SUITE_P(ContentAddressMethod, ContentAddressMethodTest, methods);
TEST(ContentAddressMethod, testParseContentAddressMethodOptException)
{
EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError);
}
/* ----------------------------------------------------------------------------
* JSON
* --------------------------------------------------------------------------*/
class ContentAddressTest : public virtual CharacterizationTest
{
std::filesystem::path unitTestData = getUnitTestData() / "content-address";
public:
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
};
using nlohmann::json;
struct ContentAddressJsonTest : ContentAddressTest,
JsonCharacterizationTest<ContentAddress>,
::testing::WithParamInterface<std::pair<std::string_view, ContentAddress>>
{};
TEST_P(ContentAddressJsonTest, from_json)
{
auto & [name, expected] = GetParam();
readJsonTest(name, expected);
}
TEST_P(ContentAddressJsonTest, to_json)
{
auto & [name, value] = GetParam();
writeJsonTest(name, value);
}
INSTANTIATE_TEST_SUITE_P(
ContentAddressJSON,
ContentAddressJsonTest,
::testing::Values(
std::pair{
"text",
ContentAddress{
.method = ContentAddressMethod::Raw::Text,
.hash = hashString(HashAlgorithm::SHA256, "asdf"),
},
},
std::pair{
"nar",
ContentAddress{
.method = ContentAddressMethod::Raw::NixArchive,
.hash = hashString(HashAlgorithm::SHA256, "qwer"),
},
}));
} // namespace nix
@@ -1,8 +0,0 @@
{
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "9vLqj0XYoFfJVmoz+ZR02i5camYE1zYSFlDicwxvsKM="
},
"method": "nar"
}
@@ -1,8 +0,0 @@
{
"hash": {
"algorithm": "sha256",
"format": "base64",
"hash": "8OTC92xYkW7CWPJGhRvqCR0U1CR6L8PhhpRGGxgW4Ts="
},
"method": "text"
}
@@ -1 +1 @@
local://?root=/foo%20bar/baz
local://?root=/foo bar/baz
@@ -1 +0,0 @@
local://?root=/foo bar/baz
+1 -538
View File
@@ -218,70 +218,6 @@ struct LambdaAdapter
}
};
class NixApiStoreTestWithRealisedPath : public nix_api_store_test_base
{
public:
StorePath * drvPath = nullptr;
nix_derivation * drv = nullptr;
Store * store = nullptr;
StorePath * outPath = nullptr;
void SetUp() override
{
nix_api_store_test_base::SetUp();
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
store = open_local_store();
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
std::stringstream buffer;
buffer << t.rdbuf();
// Replace the hardcoded system with the current system
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
drvPath = nix_add_derivation(ctx, store, drv);
assert_ctx_ok();
ASSERT_NE(drvPath, nullptr);
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath_) {
ASSERT_NE(outname, nullptr) << "Output name should not be NULL";
auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath_);
ASSERT_EQ(is_valid_path, true);
ASSERT_STREQ(outname, "out") << "Expected single 'out' output";
ASSERT_EQ(outPath, nullptr) << "Output path callback should only be called once";
outPath = nix_store_path_clone(outPath_);
}};
auto ret = nix_store_realise(
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
ASSERT_NE(outPath, nullptr) << "Derivation should have produced an output";
}
void TearDown() override
{
if (drvPath)
nix_store_path_free(drvPath);
if (outPath)
nix_store_path_free(outPath);
if (drv)
nix_derivation_free(drv);
if (store)
nix_store_free(store);
nix_api_store_test_base::TearDown();
}
};
TEST_F(nix_api_store_test_base, build_from_json)
{
// FIXME get rid of these
@@ -296,10 +232,7 @@ TEST_F(nix_api_store_test_base, build_from_json)
std::stringstream buffer;
buffer << t.rdbuf();
// Replace the hardcoded system with the current system
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
auto * drv = nix_derivation_from_json(ctx, store, buffer.str().c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
@@ -307,21 +240,15 @@ TEST_F(nix_api_store_test_base, build_from_json)
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
int callbackCount = 0;
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) {
ASSERT_NE(outname, nullptr);
ASSERT_STREQ(outname, "out");
ASSERT_NE(outPath, nullptr);
auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath);
ASSERT_EQ(is_valid_path, true);
callbackCount++;
}};
auto ret = nix_store_realise(
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
ASSERT_EQ(callbackCount, 1) << "Callback should have been invoked exactly once";
// Clean up
nix_store_path_free(drvPath);
@@ -329,468 +256,4 @@ TEST_F(nix_api_store_test_base, build_from_json)
nix_store_free(store);
}
TEST_F(nix_api_store_test_base, nix_store_realise_invalid_system)
{
// Test that nix_store_realise properly reports errors when the system is invalid
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
auto * store = open_local_store();
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
std::stringstream buffer;
buffer << t.rdbuf();
// Use an invalid system that cannot be built
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", "bogus65-bogusos");
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
auto * drvPath = nix_add_derivation(ctx, store, drv);
assert_ctx_ok();
ASSERT_NE(drvPath, nullptr);
int callbackCount = 0;
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }};
auto ret = nix_store_realise(
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
// Should fail with an error
ASSERT_NE(ret, NIX_OK);
ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build fails";
// Check that error message is set
std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
ASSERT_FALSE(errMsg.empty()) << "Error message should be set";
ASSERT_NE(errMsg.find("system"), std::string::npos) << "Error should mention system";
// Clean up
nix_store_path_free(drvPath);
nix_derivation_free(drv);
nix_store_free(store);
}
TEST_F(nix_api_store_test_base, nix_store_realise_builder_fails)
{
// Test that nix_store_realise properly reports errors when the builder fails
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
auto * store = open_local_store();
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
std::stringstream buffer;
buffer << t.rdbuf();
// Replace with current system and make builder command fail
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
jsonStr = nix::replaceStrings(jsonStr, "echo $name foo > $out", "exit 1");
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
auto * drvPath = nix_add_derivation(ctx, store, drv);
assert_ctx_ok();
ASSERT_NE(drvPath, nullptr);
int callbackCount = 0;
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }};
auto ret = nix_store_realise(
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
// Should fail with an error
ASSERT_NE(ret, NIX_OK);
ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build fails";
// Check that error message is set
std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
ASSERT_FALSE(errMsg.empty()) << "Error message should be set";
// Clean up
nix_store_path_free(drvPath);
nix_derivation_free(drv);
nix_store_free(store);
}
TEST_F(nix_api_store_test_base, nix_store_realise_builder_no_output)
{
// Test that nix_store_realise properly reports errors when builder succeeds but produces no output
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
auto * store = open_local_store();
std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")};
std::ifstream t{unitTestData / "derivation/ca/self-contained.json"};
std::stringstream buffer;
buffer << t.rdbuf();
// Replace with current system and make builder succeed but not produce output
std::string jsonStr = nix::replaceStrings(buffer.str(), "x86_64-linux", nix::settings.thisSystem.get());
jsonStr = nix::replaceStrings(jsonStr, "echo $name foo > $out", "true");
auto * drv = nix_derivation_from_json(ctx, store, jsonStr.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
auto * drvPath = nix_add_derivation(ctx, store, drv);
assert_ctx_ok();
ASSERT_NE(drvPath, nullptr);
int callbackCount = 0;
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { callbackCount++; }};
auto ret = nix_store_realise(
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
// Should fail with an error
ASSERT_NE(ret, NIX_OK);
ASSERT_EQ(callbackCount, 0) << "Callback should not be invoked when build produces no output";
// Check that error message is set
std::string errMsg = nix_err_msg(nullptr, ctx, nullptr);
ASSERT_FALSE(errMsg.empty()) << "Error message should be set";
// Clean up
nix_store_path_free(drvPath);
nix_derivation_free(drv);
nix_store_free(store);
}
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_with_outputs)
{
// Test closure computation with include_outputs on a derivation path
struct CallbackData
{
std::set<std::string> * paths;
};
std::set<std::string> closure_paths;
CallbackData data{&closure_paths};
auto ret = nix_store_get_fs_closure(
ctx,
store,
drvPath, // Use derivation path
false, // flip_direction
true, // include_outputs - include the outputs in the closure
false, // include_derivers
&data,
[](nix_c_context * context, void * userdata, const StorePath * path) {
auto * data = static_cast<CallbackData *>(userdata);
std::string path_str;
nix_store_path_name(path, OBSERVE_STRING(path_str));
auto [it, inserted] = data->paths->insert(path_str);
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
});
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// The closure should contain the derivation and its outputs
ASSERT_GE(closure_paths.size(), 2);
// Verify the output path is in the closure
std::string outPathName;
nix_store_path_name(outPath, OBSERVE_STRING(outPathName));
ASSERT_EQ(closure_paths.count(outPathName), 1);
}
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_without_outputs)
{
// Test closure computation WITHOUT include_outputs on a derivation path
struct CallbackData
{
std::set<std::string> * paths;
};
std::set<std::string> closure_paths;
CallbackData data{&closure_paths};
auto ret = nix_store_get_fs_closure(
ctx,
store,
drvPath, // Use derivation path
false, // flip_direction
false, // include_outputs - do NOT include the outputs
false, // include_derivers
&data,
[](nix_c_context * context, void * userdata, const StorePath * path) {
auto * data = static_cast<CallbackData *>(userdata);
std::string path_str;
nix_store_path_name(path, OBSERVE_STRING(path_str));
auto [it, inserted] = data->paths->insert(path_str);
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
});
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// Verify the output path is NOT in the closure
std::string outPathName;
nix_store_path_name(outPath, OBSERVE_STRING(outPathName));
ASSERT_EQ(closure_paths.count(outPathName), 0) << "Output path should not be in closure when includeOutputs=false";
}
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_flip_direction)
{
// Test closure computation with flip_direction on a derivation path
// When flip_direction=true, we get the reverse dependencies (what depends on this path)
// For a derivation, this should NOT include outputs even with include_outputs=true
struct CallbackData
{
std::set<std::string> * paths;
};
std::set<std::string> closure_paths;
CallbackData data{&closure_paths};
auto ret = nix_store_get_fs_closure(
ctx,
store,
drvPath, // Use derivation path
true, // flip_direction - get reverse dependencies
true, // include_outputs
false, // include_derivers
&data,
[](nix_c_context * context, void * userdata, const StorePath * path) {
auto * data = static_cast<CallbackData *>(userdata);
std::string path_str;
nix_store_path_name(path, OBSERVE_STRING(path_str));
auto [it, inserted] = data->paths->insert(path_str);
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
});
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// Verify the output path is NOT in the closure when direction is flipped
std::string outPathName;
nix_store_path_name(outPath, OBSERVE_STRING(outPathName));
ASSERT_EQ(closure_paths.count(outPathName), 0) << "Output path should not be in closure when flip_direction=true";
}
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_include_derivers)
{
// Test closure computation with include_derivers on an output path
// This should include the derivation that produced the output
struct CallbackData
{
std::set<std::string> * paths;
};
std::set<std::string> closure_paths;
CallbackData data{&closure_paths};
auto ret = nix_store_get_fs_closure(
ctx,
store,
outPath, // Use output path (not derivation)
false, // flip_direction
false, // include_outputs
true, // include_derivers - include the derivation
&data,
[](nix_c_context * context, void * userdata, const StorePath * path) {
auto * data = static_cast<CallbackData *>(userdata);
std::string path_str;
nix_store_path_name(path, OBSERVE_STRING(path_str));
auto [it, inserted] = data->paths->insert(path_str);
ASSERT_TRUE(inserted) << "Duplicate path in closure: " << path_str;
});
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// Verify the derivation path is in the closure
// Deriver is nasty stateful, and this assertion is only guaranteed because
// we're using an empty store as our starting point. Otherwise, if the
// output happens to exist, the deriver could be anything.
std::string drvPathName;
nix_store_path_name(drvPath, OBSERVE_STRING(drvPathName));
ASSERT_EQ(closure_paths.count(drvPathName), 1) << "Derivation should be in closure when include_derivers=true";
}
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_realise_output_ordering)
{
// Test that nix_store_realise returns outputs in alphabetical order by output name.
// This test uses a CA derivation with 10 outputs in randomized input order
// to verify that the callback order is deterministic and alphabetical.
nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
nix::settings.substituters = {};
auto * store = open_local_store();
// Create a CA derivation with 10 outputs using proper placeholders
auto outa_ph = nix::hashPlaceholder("outa");
auto outb_ph = nix::hashPlaceholder("outb");
auto outc_ph = nix::hashPlaceholder("outc");
auto outd_ph = nix::hashPlaceholder("outd");
auto oute_ph = nix::hashPlaceholder("oute");
auto outf_ph = nix::hashPlaceholder("outf");
auto outg_ph = nix::hashPlaceholder("outg");
auto outh_ph = nix::hashPlaceholder("outh");
auto outi_ph = nix::hashPlaceholder("outi");
auto outj_ph = nix::hashPlaceholder("outj");
std::string drvJson = R"({
"version": 3,
"name": "multi-output-test",
"system": ")" + nix::settings.thisSystem.get()
+ R"(",
"builder": "/bin/sh",
"args": ["-c", "echo a > $outa; echo b > $outb; echo c > $outc; echo d > $outd; echo e > $oute; echo f > $outf; echo g > $outg; echo h > $outh; echo i > $outi; echo j > $outj"],
"env": {
"builder": "/bin/sh",
"name": "multi-output-test",
"system": ")" + nix::settings.thisSystem.get()
+ R"(",
"outf": ")" + outf_ph
+ R"(",
"outd": ")" + outd_ph
+ R"(",
"outi": ")" + outi_ph
+ R"(",
"oute": ")" + oute_ph
+ R"(",
"outh": ")" + outh_ph
+ R"(",
"outc": ")" + outc_ph
+ R"(",
"outb": ")" + outb_ph
+ R"(",
"outg": ")" + outg_ph
+ R"(",
"outj": ")" + outj_ph
+ R"(",
"outa": ")" + outa_ph
+ R"("
},
"inputDrvs": {},
"inputSrcs": [],
"outputs": {
"outd": { "hashAlgo": "sha256", "method": "nar" },
"outf": { "hashAlgo": "sha256", "method": "nar" },
"outg": { "hashAlgo": "sha256", "method": "nar" },
"outb": { "hashAlgo": "sha256", "method": "nar" },
"outc": { "hashAlgo": "sha256", "method": "nar" },
"outi": { "hashAlgo": "sha256", "method": "nar" },
"outj": { "hashAlgo": "sha256", "method": "nar" },
"outh": { "hashAlgo": "sha256", "method": "nar" },
"outa": { "hashAlgo": "sha256", "method": "nar" },
"oute": { "hashAlgo": "sha256", "method": "nar" }
}
})";
auto * drv = nix_derivation_from_json(ctx, store, drvJson.c_str());
assert_ctx_ok();
ASSERT_NE(drv, nullptr);
auto * drvPath = nix_add_derivation(ctx, store, drv);
assert_ctx_ok();
ASSERT_NE(drvPath, nullptr);
// Realise the derivation - capture the order outputs are returned
std::map<std::string, nix::StorePath> outputs;
std::vector<std::string> output_order;
auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) {
ASSERT_NE(outname, nullptr);
ASSERT_NE(outPath, nullptr);
output_order.push_back(outname);
outputs.emplace(outname, outPath->path);
}};
auto ret = nix_store_realise(
ctx, store, drvPath, static_cast<void *>(&cb), decltype(cb)::call_void<const char *, const StorePath *>);
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
ASSERT_EQ(outputs.size(), 10);
// Verify outputs are returned in alphabetical order by output name
std::vector<std::string> expected_order = {
"outa", "outb", "outc", "outd", "oute", "outf", "outg", "outh", "outi", "outj"};
ASSERT_EQ(output_order, expected_order) << "Outputs should be returned in alphabetical order by output name";
// Now compute closure with include_outputs and collect paths in order
struct CallbackData
{
std::vector<std::string> * paths;
};
std::vector<std::string> closure_paths;
CallbackData data{&closure_paths};
ret = nix_store_get_fs_closure(
ctx,
store,
drvPath,
false, // flip_direction
true, // include_outputs - include the outputs in the closure
false, // include_derivers
&data,
[](nix_c_context * context, void * userdata, const StorePath * path) {
auto * data = static_cast<CallbackData *>(userdata);
std::string path_str;
nix_store_path_name(path, OBSERVE_STRING(path_str));
data->paths->push_back(path_str);
});
assert_ctx_ok();
ASSERT_EQ(ret, NIX_OK);
// Should contain at least the derivation and 10 outputs
ASSERT_GE(closure_paths.size(), 11);
// Verify all outputs are present in the closure
for (const auto & [outname, outPath] : outputs) {
std::string outPathName = store->ptr->printStorePath(outPath);
bool found = false;
for (const auto & p : closure_paths) {
// nix_store_path_name returns just the name part, so match against full path name
if (outPathName.find(p) != std::string::npos) {
found = true;
break;
}
}
ASSERT_TRUE(found) << "Output " << outname << " (" << outPathName << ") not found in closure";
}
nix_store_path_free(drvPath);
nix_derivation_free(drv);
nix_store_free(store);
}
TEST_F(NixApiStoreTestWithRealisedPath, nix_store_get_fs_closure_error_propagation)
{
// Test that errors in the callback abort the closure computation
struct CallbackData
{
int * count;
};
int call_count = 0;
CallbackData data{&call_count};
auto ret = nix_store_get_fs_closure(
ctx,
store,
drvPath, // Use derivation path
false, // flip_direction
true, // include_outputs
false, // include_derivers
&data,
[](nix_c_context * context, void * userdata, const StorePath * path) {
auto * data = static_cast<CallbackData *>(userdata);
(*data->count)++;
// Set an error immediately
nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Test error");
});
// Should have aborted with error
ASSERT_EQ(ret, NIX_ERR_UNKNOWN);
ASSERT_EQ(call_count, 1); // Should have been called exactly once, then aborted
}
} // namespace nixC
-143
View File
@@ -1,6 +1,4 @@
#include "nix/store/references.hh"
#include "nix/store/path-references.hh"
#include "nix/util/memory-source-accessor.hh"
#include <gtest/gtest.h>
@@ -81,145 +79,4 @@ TEST(references, scan)
}
}
TEST(references, scanForReferencesDeep)
{
using File = MemorySourceAccessor::File;
// Create store paths to search for
StorePath path1{"dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo"};
StorePath path2{"zc842j0rz61mjsp3h3wp5ly71ak6qgdn-bar"};
StorePath path3{"a5cn2i4b83gnsm60d38l3kgb8qfplm11-baz"};
StorePathSet refs{path1, path2, path3};
std::string_view hash1 = path1.hashPart();
std::string_view hash2 = path2.hashPart();
std::string_view hash3 = path3.hashPart();
// Create an in-memory file system with various reference patterns
auto accessor = make_ref<MemorySourceAccessor>();
accessor->root = File::Directory{
.contents{
{
// file1.txt: contains hash1
"file1.txt",
File::Regular{
.contents = "This file references " + hash1 + " in its content",
},
},
{
// file2.txt: contains hash2 and hash3
"file2.txt",
File::Regular{
.contents = "Multiple refs: " + hash2 + " and also " + hash3,
},
},
{
// file3.txt: contains no references
"file3.txt",
File::Regular{
.contents = "This file has no store path references at all",
},
},
{
// subdir: a subdirectory
"subdir",
File::Directory{
.contents{
{
// subdir/file4.txt: contains hash1 again
"file4.txt",
File::Regular{
.contents = "Subdirectory file with " + hash1,
},
},
},
},
},
{
// link1: a symlink that contains a reference in its target
"link1",
File::Symlink{
.target = hash2 + "-target",
},
},
},
};
// Test the callback-based API
{
std::map<CanonPath, StorePathSet> foundRefs;
scanForReferencesDeep(*accessor, CanonPath::root, refs, [&](FileRefScanResult result) {
foundRefs[std::move(result.filePath)] = std::move(result.foundRefs);
});
// Verify we found the expected references
EXPECT_EQ(foundRefs.size(), 4); // file1, file2, file4, link1
// Check file1.txt found path1
{
CanonPath f1Path("/file1.txt");
auto it = foundRefs.find(f1Path);
ASSERT_TRUE(it != foundRefs.end());
EXPECT_EQ(it->second.size(), 1);
EXPECT_TRUE(it->second.count(path1));
}
// Check file2.txt found path2 and path3
{
CanonPath f2Path("/file2.txt");
auto it = foundRefs.find(f2Path);
ASSERT_TRUE(it != foundRefs.end());
EXPECT_EQ(it->second.size(), 2);
EXPECT_TRUE(it->second.count(path2));
EXPECT_TRUE(it->second.count(path3));
}
// Check file3.txt is not in results (no refs)
{
CanonPath f3Path("/file3.txt");
EXPECT_FALSE(foundRefs.count(f3Path));
}
// Check subdir/file4.txt found path1
{
CanonPath f4Path("/subdir/file4.txt");
auto it = foundRefs.find(f4Path);
ASSERT_TRUE(it != foundRefs.end());
EXPECT_EQ(it->second.size(), 1);
EXPECT_TRUE(it->second.count(path1));
}
// Check symlink found path2
{
CanonPath linkPath("/link1");
auto it = foundRefs.find(linkPath);
ASSERT_TRUE(it != foundRefs.end());
EXPECT_EQ(it->second.size(), 1);
EXPECT_TRUE(it->second.count(path2));
}
}
// Test the map-based convenience API
{
auto results = scanForReferencesDeep(*accessor, CanonPath::root, refs);
EXPECT_EQ(results.size(), 4); // file1, file2, file4, link1
// Verify all expected files are in the results
EXPECT_TRUE(results.count(CanonPath("/file1.txt")));
EXPECT_TRUE(results.count(CanonPath("/file2.txt")));
EXPECT_TRUE(results.count(CanonPath("/subdir/file4.txt")));
EXPECT_TRUE(results.count(CanonPath("/link1")));
EXPECT_FALSE(results.count(CanonPath("/file3.txt")));
// Verify the references found in each file are correct
EXPECT_EQ(results.at(CanonPath("/file1.txt")), StorePathSet{path1});
EXPECT_EQ(results.at(CanonPath("/file2.txt")), StorePathSet({path2, path3}));
EXPECT_EQ(results.at(CanonPath("/subdir/file4.txt")), StorePathSet{path1});
EXPECT_EQ(results.at(CanonPath("/link1")), StorePathSet{path2});
}
}
} // namespace nix
-50
View File
@@ -70,25 +70,6 @@ INSTANTIATE_TEST_SUITE_P(
},
"with_profile_and_region",
},
ParsedS3URLTestCase{
"s3://my-bucket/my-key.txt?versionId=abc123xyz",
{
.bucket = "my-bucket",
.key = {"my-key.txt"},
.versionId = "abc123xyz",
},
"with_versionId",
},
ParsedS3URLTestCase{
"s3://bucket/path/to/object?region=eu-west-1&versionId=version456",
{
.bucket = "bucket",
.key = {"path", "to", "object"},
.region = "eu-west-1",
.versionId = "version456",
},
"with_region_and_versionId",
},
ParsedS3URLTestCase{
"s3://bucket/key?endpoint=https://minio.local&scheme=http",
{
@@ -241,37 +222,6 @@ INSTANTIATE_TEST_SUITE_P(
},
"https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt",
"complex_path_and_region",
},
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "my-bucket",
.key = {"my-key.txt"},
.versionId = "abc123xyz",
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"},
.path = {"", "my-bucket", "my-key.txt"},
.query = {{"versionId", "abc123xyz"}},
},
"https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt?versionId=abc123xyz",
"with_versionId",
},
S3ToHttpsConversionTestCase{
ParsedS3URL{
.bucket = "versioned-bucket",
.key = {"path", "to", "object"},
.region = "eu-west-1",
.versionId = "version456",
},
ParsedURL{
.scheme = "https",
.authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"},
.path = {"", "versioned-bucket", "path", "to", "object"},
.query = {{"versionId", "version456"}},
},
"https://s3.eu-west-1.amazonaws.com/versioned-bucket/path/to/object?versionId=version456",
"with_region_and_versionId",
}),
[](const ::testing::TestParamInfo<S3ToHttpsConversionTestCase> & info) { return info.param.description; });
+1 -4
View File
@@ -100,12 +100,9 @@ URI_TEST(local_1, localExample_1)
URI_TEST(local_2, localExample_2)
/* Test path with encoded spaces */
/* Test path with spaces */
URI_TEST(local_3, localExample_3)
/* Test path with spaces that are improperly not encoded */
URI_TEST_READ(local_3_no_percent, localExample_3)
URI_TEST_READ(local_shorthand_1, localExample_1)
URI_TEST_READ(local_shorthand_2, localExample_2)
+3 -5
View File
@@ -76,10 +76,9 @@ std::optional<std::string> BinaryCacheStore::getNixCacheInfo()
return getFile(cacheInfoFile);
}
void BinaryCacheStore::upsertFile(
const std::string & path, std::string && data, const std::string & mimeType, uint64_t sizeHint)
void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType)
{
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType, sizeHint);
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
}
void BinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
@@ -275,8 +274,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
upsertFile(
narInfo->url,
std::make_shared<std::fstream>(fnTemp, std::ios_base::in | std::ios_base::binary),
"application/x-nix-nar",
narInfo->fileSize);
"application/x-nix-nar");
} else
stats.narWriteAverted++;
@@ -239,6 +239,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
: buildMode == bmCheck ? "checking outputs of '%s'"
: "building '%s'",
worker.store.printStorePath(drvPath));
fmt("building '%s'", worker.store.printStorePath(drvPath));
#ifndef _WIN32 // TODO enable build hook on Windows
if (hook)
msg += fmt(" on '%s'", hook->machineName);
@@ -677,15 +678,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild()
{
builder.reset();
StorePathSet outputPaths;
/* In the check case we install no store objects, and so
`builtOutputs` is empty. However, per issue #14287, there is
an expectation that the post-build hook is still executed.
(This is useful for e.g. logging successful deterministic rebuilds.)
In order to make that work, in the check case just load the
(preexisting) infos from scratch, rather than relying on what
`DerivationBuilder` returned to us. */
for (auto & [_, output] : buildMode == bmCheck ? checkPathValidity(initialOutputs).second : builtOutputs) {
for (auto & [_, output] : builtOutputs) {
// for sake of `bmRepair`
worker.markContentsGood(output.outPath);
outputPaths.insert(output.outPath);
+1 -1
View File
@@ -25,7 +25,7 @@ DesugaredEnv DesugaredEnv::create(
if (drv.structuredAttrs) {
auto json = drv.structuredAttrs->prepareStructuredAttrs(store, drvOptions, inputPaths, drv.outputs);
res.atFileEnvPair("NIX_ATTRS_SH_FILE", ".attrs.sh") = StructuredAttrs::writeShell(json);
res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = static_cast<nlohmann::json>(std::move(json)).dump();
res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = json.dump();
} else {
/* In non-structured mode, set all bindings either directory in the
environment or via a file, as specified by
+3 -5
View File
@@ -268,18 +268,16 @@ Goal::Co PathSubstitutionGoal::tryToRun(
try {
promise.get_future().get();
} catch (std::exception & e) {
printError(e.what());
/* Cause the parent build to fail unless --fallback is given,
or the substitute has disappeared. The latter case behaves
the same as the substitute never having existed in the
first place. */
try {
throw;
} catch (SubstituteGone & sg) {
/* Missing NARs are expected when they've been garbage collected.
This is not a failure, so log as a warning instead of an error. */
logWarning({.msg = sg.info().msg});
} catch (SubstituteGone &) {
} catch (...) {
printError(e.what());
substituterFailed = true;
}
-34
View File
@@ -1,7 +1,6 @@
#include "nix/util/args.hh"
#include "nix/store/content-address.hh"
#include "nix/util/split.hh"
#include "nix/util/json-utils.hh"
namespace nix {
@@ -301,36 +300,3 @@ Hash ContentAddressWithReferences::getHash() const
}
} // namespace nix
namespace nlohmann {
using namespace nix;
ContentAddressMethod adl_serializer<ContentAddressMethod>::from_json(const json & json)
{
return ContentAddressMethod::parse(getString(json));
}
void adl_serializer<ContentAddressMethod>::to_json(json & json, const ContentAddressMethod & m)
{
json = m.render();
}
ContentAddress adl_serializer<ContentAddress>::from_json(const json & json)
{
auto obj = getObject(json);
return {
.method = adl_serializer<ContentAddressMethod>::from_json(valueAt(obj, "method")),
.hash = valueAt(obj, "hash"),
};
}
void adl_serializer<ContentAddress>::to_json(json & json, const ContentAddress & ca)
{
json = {
{"method", ca.method},
{"hash", ca.hash},
};
}
} // namespace nlohmann
+1 -1
View File
@@ -1031,7 +1031,7 @@ void processConnection(ref<Store> store, FdSource && from, FdSink && to, Trusted
auto [protoVersion, features] =
WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, WorkerProto::allFeatures);
if (protoVersion < MINIMUM_PROTOCOL_VERSION)
if (protoVersion < 256 + 18)
throw Error("the Nix client version is too old");
WorkerProto::BasicServerConnection conn;
+20 -42
View File
@@ -22,9 +22,9 @@ getStringAttr(const StringMap & env, const StructuredAttrs * parsed, const std::
if (i == parsed->structuredAttrs.end())
return {};
else {
if (!i->second.is_string())
if (!i->is_string())
throw Error("attribute '%s' of must be a string", name);
return i->second.get<std::string>();
return i->get<std::string>();
}
} else {
auto i = env.find(name);
@@ -42,9 +42,9 @@ static bool getBoolAttr(const StringMap & env, const StructuredAttrs * parsed, c
if (i == parsed->structuredAttrs.end())
return def;
else {
if (!i->second.is_boolean())
if (!i->is_boolean())
throw Error("attribute '%s' must be a Boolean", name);
return i->second.get<bool>();
return i->get<bool>();
}
} else {
auto i = env.find(name);
@@ -63,11 +63,10 @@ getStringsAttr(const StringMap & env, const StructuredAttrs * parsed, const std:
if (i == parsed->structuredAttrs.end())
return {};
else {
if (!i->second.is_array())
if (!i->is_array())
throw Error("attribute '%s' must be a list of strings", name);
auto & a = getArray(i->second);
Strings res;
for (auto j = a.begin(); j != a.end(); ++j) {
for (auto j = i->begin(); j != i->end(); ++j) {
if (!j->is_string())
throw Error("attribute '%s' must be a list of strings", name);
res.push_back(j->get<std::string>());
@@ -117,29 +116,27 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
DerivationOptions defaults = {};
if (shouldWarn && parsed) {
auto & structuredAttrs = parsed->structuredAttrs;
if (get(structuredAttrs, "allowedReferences")) {
if (get(parsed->structuredAttrs, "allowedReferences")) {
warn(
"'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead");
}
if (get(structuredAttrs, "allowedRequisites")) {
if (get(parsed->structuredAttrs, "allowedRequisites")) {
warn(
"'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead");
}
if (get(structuredAttrs, "disallowedRequisites")) {
if (get(parsed->structuredAttrs, "disallowedRequisites")) {
warn(
"'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead");
}
if (get(structuredAttrs, "disallowedReferences")) {
if (get(parsed->structuredAttrs, "disallowedReferences")) {
warn(
"'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead");
}
if (get(structuredAttrs, "maxSize")) {
if (get(parsed->structuredAttrs, "maxSize")) {
warn(
"'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead");
}
if (get(structuredAttrs, "maxClosureSize")) {
if (get(parsed->structuredAttrs, "maxClosureSize")) {
warn(
"'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead");
}
@@ -148,15 +145,11 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
return {
.outputChecks = [&]() -> OutputChecksVariant {
if (parsed) {
auto & structuredAttrs = parsed->structuredAttrs;
std::map<std::string, OutputChecks> res;
if (auto * outputChecks = get(structuredAttrs, "outputChecks")) {
for (auto & [outputName, output_] : getObject(*outputChecks)) {
if (auto outputChecks = get(parsed->structuredAttrs, "outputChecks")) {
for (auto & [outputName, output] : getObject(*outputChecks)) {
OutputChecks checks;
auto & output = getObject(output_);
if (auto maxSize = get(output, "maxSize"))
checks.maxSize = maxSize->get<uint64_t>();
@@ -202,9 +195,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
std::map<std::string, bool> res;
if (parsed) {
auto & structuredAttrs = parsed->structuredAttrs;
if (auto * udr = get(structuredAttrs, "unsafeDiscardReferences")) {
if (auto udr = get(parsed->structuredAttrs, "unsafeDiscardReferences")) {
for (auto & [outputName, output] : getObject(*udr)) {
if (!output.is_boolean())
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' must be a Boolean", outputName);
@@ -235,7 +226,7 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
std::map<std::string, StringSet> ret;
if (parsed) {
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
auto e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
if (!e || !e->is_object())
return ret;
for (auto & [key, value] : getObject(*e)) {
@@ -342,10 +333,8 @@ namespace nlohmann {
using namespace nix;
DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json_)
DerivationOptions adl_serializer<DerivationOptions>::from_json(const json & json)
{
auto & json = getObject(json_);
return {
.outputChecks = [&]() -> OutputChecksVariant {
auto outputChecks = getObject(valueAt(json, "outputChecks"));
@@ -408,24 +397,13 @@ void adl_serializer<DerivationOptions>::to_json(json & json, const DerivationOpt
json["allowSubstitutes"] = o.allowSubstitutes;
}
template<typename T>
static inline std::optional<T> ptrToOwned(const json * ptr)
DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json)
{
if (ptr)
return std::optional{*ptr};
else
return std::nullopt;
}
DerivationOptions::OutputChecks adl_serializer<DerivationOptions::OutputChecks>::from_json(const json & json_)
{
auto & json = getObject(json_);
return {
.ignoreSelfRefs = getBoolean(valueAt(json, "ignoreSelfRefs")),
.allowedReferences = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedReferences"))),
.allowedReferences = nullableValueAt(json, "allowedReferences"),
.disallowedReferences = getStringSet(valueAt(json, "disallowedReferences")),
.allowedRequisites = ptrToOwned<StringSet>(getNullable(valueAt(json, "allowedRequisites"))),
.allowedRequisites = nullableValueAt(json, "allowedRequisites"),
.disallowedRequisites = getStringSet(valueAt(json, "disallowedRequisites")),
};
}
+13 -1
View File
@@ -148,7 +148,7 @@ struct DummyStoreImpl : DummyStore
/**
* The dummy store is incapable of *not* trusting! :)
*/
std::optional<TrustedFlag> isTrustedClient() override
virtual std::optional<TrustedFlag> isTrustedClient() override
{
return Trusted;
}
@@ -258,6 +258,18 @@ struct DummyStoreImpl : DummyStore
});
}
void narFromPath(const StorePath & path, Sink & sink) override
{
bool visited = contents.cvisit(path, [&](const auto & kv) {
const auto & [info, accessor] = kv.second;
SourcePath sourcePath(accessor);
dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive);
});
if (!visited)
throw Error("path '%s' is not valid", printStorePath(path));
}
void queryRealisationUncached(
const DrvOutput & drvOutput, Callback<std::shared_ptr<const UnkeyedRealisation>> callback) noexcept override
{
+12 -19
View File
@@ -384,30 +384,28 @@ struct curlFileTransfer : public FileTransfer
if (settings.downloadSpeed.get() > 0)
curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
if (request.method == HttpMethod::HEAD)
if (request.head)
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
if (request.method == HttpMethod::DELETE)
curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
if (request.data) {
if (request.method == HttpMethod::POST) {
if (request.post)
curl_easy_setopt(req, CURLOPT_POST, 1L);
curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->length());
} else if (request.method == HttpMethod::PUT) {
else
curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
} else {
unreachable();
}
curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper);
curl_easy_setopt(req, CURLOPT_READDATA, this);
curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length());
curl_easy_setopt(req, CURLOPT_SEEKFUNCTION, seekCallbackWrapper);
curl_easy_setopt(req, CURLOPT_SEEKDATA, this);
}
if (settings.caFile != "")
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
if (request.verifyTLS) {
if (settings.caFile != "")
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str());
} else {
curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0);
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
}
#if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
@@ -624,7 +622,7 @@ struct curlFileTransfer : public FileTransfer
void quit()
{
quitting = true;
/* We wil not be processing any more incoming requests */
/* We wil not be processing any more incomming requests */
while (!incoming.empty())
incoming.pop();
}
@@ -924,11 +922,6 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
return enqueueFileTransfer(request).get();
}
FileTransferResult FileTransfer::deleteResource(const FileTransferRequest & request)
{
return enqueueFileTransfer(request).get();
}
void FileTransfer::download(
FileTransferRequest && request, Sink & sink, std::function<void(FileTransferResult)> resultCallback)
{
+3 -2
View File
@@ -5,7 +5,6 @@
#include "nix/util/finally.hh"
#include "nix/util/unix-domain-socket.hh"
#include "nix/util/signals.hh"
#include "nix/util/util.hh"
#include "nix/store/posix-fs-canonicalise.hh"
#include "store-config-private.hh"
@@ -907,7 +906,9 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
#endif
;
printInfo("note: currently hard linking saves %s", renderSize(unsharedSize - actualSize - overhead));
printInfo(
"note: currently hard linking saves %.2f MiB",
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
/* While we're at it, vacuum the database. */
+1 -1
View File
@@ -150,7 +150,7 @@ std::vector<Path> getUserConfigFiles()
return files;
}
unsigned int Settings::getDefaultCores()
unsigned int Settings::getDefaultCores() const
{
const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency());
const unsigned int maxCPU = getMaxCPU();
+200 -187
View File
@@ -51,211 +51,224 @@ std::string HttpBinaryCacheStoreConfig::doc()
;
}
HttpBinaryCacheStore::HttpBinaryCacheStore(ref<Config> config)
: Store{*config} // TODO it will actually mutate the configuration
, BinaryCacheStore{*config}
, config{config}
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
diskCache = getNarInfoDiskCache();
}
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
void HttpBinaryCacheStore::init()
{
// FIXME: do this lazily?
// For consistent cache key handling, use the reference without parameters
// This matches what's used in Store::queryPathInfo() lookups
auto cacheKey = config->getReference().render(/*withParams=*/false);
Sync<State> _state;
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) {
config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
config->priority.setDefault(cacheInfo->priority);
} else {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string());
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config)
: Store{*config} // TODO it will actually mutate the configuration
, BinaryCacheStore{*config}
, config{config}
{
diskCache = getNarInfoDiskCache();
}
void init() override
{
// FIXME: do this lazily?
// For consistent cache key handling, use the reference without parameters
// This matches what's used in Store::queryPathInfo() lookups
auto cacheKey = config->getReference().render(/*withParams=*/false);
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheKey)) {
config->wantMassQuery.setDefault(cacheInfo->wantMassQuery);
config->priority.setDefault(cacheInfo->priority);
} else {
try {
BinaryCacheStore::init();
} catch (UploadToHTTP &) {
throw Error("'%s' does not appear to be a binary cache", config->cacheUri.to_string());
}
diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority);
}
diskCache->createCache(cacheKey, config->storeDir, config->wantMassQuery, config->priority);
}
}
std::optional<std::string> HttpBinaryCacheStore::getCompressionMethod(const std::string & path)
{
if (hasSuffix(path, ".narinfo") && !config->narinfoCompression.get().empty())
return config->narinfoCompression;
else if (hasSuffix(path, ".ls") && !config->lsCompression.get().empty())
return config->lsCompression;
else if (hasPrefix(path, "log/") && !config->logCompression.get().empty())
return config->logCompression;
else
return std::nullopt;
}
void HttpBinaryCacheStore::maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t);
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
void HttpBinaryCacheStore::checkEnabled()
{
auto state(_state.lock());
if (state->enabled)
return;
if (std::chrono::steady_clock::now() > state->disabledUntil) {
state->enabled = true;
debug("re-enabling binary cache '%s'", config->getHumanReadableURI());
return;
}
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
}
bool HttpBinaryCacheStore::fileExists(const std::string & path)
{
checkEnabled();
try {
FileTransferRequest request(makeRequest(path));
request.method = HttpMethod::HEAD;
getFileTransfer()->download(request);
return true;
} catch (FileTransferError & e) {
/* S3 buckets return 403 if a file doesn't exist and the
bucket is unlistable, so treat 403 as 404. */
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return false;
maybeDisable();
throw;
}
}
void HttpBinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint)
{
auto req = makeRequest(path);
req.method = HttpMethod::PUT;
auto data = StreamToSourceAdapter(istream).drain();
auto compressionMethod = getCompressionMethod(path);
if (compressionMethod) {
data = compress(*compressionMethod, data);
req.headers.emplace_back("Content-Encoding", *compressionMethod);
}
req.data = std::move(data);
req.mimeType = mimeType;
protected:
try {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
UploadToHTTP err(e.message());
err.addTrace({}, "while uploading to HTTP binary cache at '%s'", config->cacheUri.to_string());
throw err;
}
}
FileTransferRequest HttpBinaryCacheStore::makeRequest(std::string_view path)
{
/* Otherwise the last path fragment will get discarded. */
auto cacheUriWithTrailingSlash = config->cacheUri;
if (!cacheUriWithTrailingSlash.path.empty())
cacheUriWithTrailingSlash.path.push_back("");
/* path is not a path, but a full relative or absolute
URL, e.g. we've seen in the wild NARINFO files have a URL
field which is
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj`
(note the query param) and that gets passed here. */
auto result = parseURLRelative(path, cacheUriWithTrailingSlash);
/* For S3 URLs, preserve query parameters from the base URL when the
relative path doesn't have its own query parameters. This is needed
to preserve S3-specific parameters like endpoint and region. */
if (config->cacheUri.scheme == "s3" && result.query.empty()) {
result.query = config->cacheUri.query;
void maybeDisable()
{
auto state(_state.lock());
if (state->enabled && settings.tryFallback) {
int t = 60;
printError("disabling binary cache '%s' for %s seconds", config->getHumanReadableURI(), t);
state->enabled = false;
state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t);
}
}
return FileTransferRequest(result);
}
void HttpBinaryCacheStore::getFile(const std::string & path, Sink & sink)
{
checkEnabled();
auto request(makeRequest(path));
try {
getFileTransfer()->download(std::move(request), sink);
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI());
maybeDisable();
throw;
void checkEnabled()
{
auto state(_state.lock());
if (state->enabled)
return;
if (std::chrono::steady_clock::now() > state->disabledUntil) {
state->enabled = true;
debug("re-enabling binary cache '%s'", config->getHumanReadableURI());
return;
}
throw SubstituterDisabled("substituter '%s' is disabled", config->getHumanReadableURI());
}
}
void HttpBinaryCacheStore::getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
try {
bool fileExists(const std::string & path) override
{
checkEnabled();
try {
FileTransferRequest request(makeRequest(path));
request.head = true;
getFileTransfer()->download(request);
return true;
} catch (FileTransferError & e) {
/* S3 buckets return 403 if a file doesn't exist and the
bucket is unlistable, so treat 403 as 404. */
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return false;
maybeDisable();
throw;
}
}
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto req = makeRequest(path);
auto data = StreamToSourceAdapter(istream).drain();
// Determine compression method based on file type
std::string compressionMethod;
if (hasSuffix(path, ".narinfo"))
compressionMethod = config->narinfoCompression;
else if (hasSuffix(path, ".ls"))
compressionMethod = config->lsCompression;
else if (hasPrefix(path, "log/"))
compressionMethod = config->logCompression;
// Apply compression if configured
if (!compressionMethod.empty()) {
data = compress(compressionMethod, data);
req.headers.emplace_back("Content-Encoding", compressionMethod);
}
req.data = std::move(data);
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);
} catch (FileTransferError & e) {
throw UploadToHTTP(
"while uploading to HTTP binary cache at '%s': %s", config->cacheUri.to_string(), e.msg());
}
}
FileTransferRequest makeRequest(const std::string & path)
{
/* Otherwise the last path fragment will get discarded. */
auto cacheUriWithTrailingSlash = config->cacheUri;
if (!cacheUriWithTrailingSlash.path.empty())
cacheUriWithTrailingSlash.path.push_back("");
/* path is not a path, but a full relative or absolute
URL, e.g. we've seen in the wild NARINFO files have a URL
field which is
`nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj`
(note the query param) and that gets passed here. */
auto result = parseURLRelative(path, cacheUriWithTrailingSlash);
/* For S3 URLs, preserve query parameters from the base URL when the
relative path doesn't have its own query parameters. This is needed
to preserve S3-specific parameters like endpoint and region. */
if (config->cacheUri.scheme == "s3" && result.query.empty()) {
result.query = config->cacheUri.query;
}
return FileTransferRequest(result);
}
void getFile(const std::string & path, Sink & sink) override
{
checkEnabled();
auto request(makeRequest(path));
getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound
|| e.error == FileTransfer::Forbidden)
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
} catch (...) {
callbackPtr->rethrow();
}
}});
} catch (...) {
callbackPtr->rethrow();
return;
try {
getFileTransfer()->download(std::move(request), sink);
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile(
"file '%s' does not exist in binary cache '%s'", path, config->getHumanReadableURI());
maybeDisable();
throw;
}
}
}
std::optional<std::string> HttpBinaryCacheStore::getNixCacheInfo()
{
try {
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
return result.data;
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound)
return std::nullopt;
maybeDisable();
throw;
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override
{
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
try {
checkEnabled();
auto request(makeRequest(path));
getFileTransfer()->enqueueFileTransfer(
request, {[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(std::move(result.get().data));
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return (*callbackPtr)({});
maybeDisable();
callbackPtr->rethrow();
} catch (...) {
callbackPtr->rethrow();
}
}});
} catch (...) {
callbackPtr->rethrow();
return;
}
}
}
/**
* This isn't actually necessary read only. We support "upsert" now, so we
* have a notion of authentication via HTTP POST/PUT.
*
* For now, we conservatively say we don't know.
*
* \todo try to expose our HTTP authentication status.
*/
std::optional<TrustedFlag> HttpBinaryCacheStore::isTrustedClient()
{
return std::nullopt;
}
std::optional<std::string> getNixCacheInfo() override
{
try {
auto result = getFileTransfer()->download(makeRequest(cacheInfoFile));
return result.data;
} catch (FileTransferError & e) {
if (e.error == FileTransfer::NotFound)
return std::nullopt;
maybeDisable();
throw;
}
}
/**
* This isn't actually necessary read only. We support "upsert" now, so we
* have a notion of authentication via HTTP POST/PUT.
*
* For now, we conservatively say we don't know.
*
* \todo try to expose our HTTP authentication status.
*/
std::optional<TrustedFlag> isTrustedClient() override
{
return std::nullopt;
}
};
ref<Store> HttpBinaryCacheStore::Config::openStore() const
{
@@ -101,27 +101,13 @@ public:
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) = 0;
const std::string & path, std::shared_ptr<std::basic_iostream<char>> istream, const std::string & mimeType) = 0;
void upsertFile(
const std::string & path,
// FIXME: use std::string_view
std::string && data,
const std::string & mimeType,
uint64_t sizeHint);
void upsertFile(
const std::string & path,
// FIXME: use std::string_view
std::string && data,
const std::string & mimeType)
{
auto size = data.size();
upsertFile(path, std::move(data), mimeType, size);
}
const std::string & mimeType);
/**
* Dump the contents of the specified file to a sink.
@@ -6,7 +6,6 @@
#include "nix/store/path.hh"
#include "nix/util/file-content-address.hh"
#include "nix/util/variant-wrapper.hh"
#include "nix/util/json-impls.hh"
namespace nix {
@@ -309,15 +308,4 @@ struct ContentAddressWithReferences
Hash getHash() const;
};
template<>
struct json_avoids_null<ContentAddressMethod> : std::true_type
{};
template<>
struct json_avoids_null<ContentAddress> : std::true_type
{};
} // namespace nix
JSON_IMPL(nix::ContentAddressMethod)
JSON_IMPL(nix::ContentAddress)
+5 -36
View File
@@ -83,17 +83,6 @@ extern FileTransferSettings fileTransferSettings;
extern const unsigned int RETRY_TIME_MS_DEFAULT;
/**
* HTTP methods supported by FileTransfer.
*/
enum struct HttpMethod {
GET,
PUT,
HEAD,
POST,
DELETE,
};
/**
* Username and optional password for HTTP basic authentication.
* These are used with curl's CURLOPT_USERNAME and CURLOPT_PASSWORD options
@@ -110,7 +99,9 @@ struct FileTransferRequest
VerbatimURL uri;
Headers headers;
std::string expectedETag;
HttpMethod method = HttpMethod::GET;
bool verifyTLS = true;
bool head = false;
bool post = false;
size_t tries = fileTransferSettings.tries;
unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
ActivityId parentAct;
@@ -137,31 +128,14 @@ struct FileTransferRequest
{
}
/**
* Returns the verb root for logging purposes.
* The returned string is intended to be concatenated with "ing" to form the gerund,
* e.g., "download" + "ing" -> "downloading", "upload" + "ing" -> "uploading".
*/
std::string verb() const
{
switch (method) {
case HttpMethod::HEAD:
case HttpMethod::GET:
return "download";
case HttpMethod::PUT:
case HttpMethod::POST:
assert(data);
return "upload";
case HttpMethod::DELETE:
return "delet";
}
unreachable();
return data ? "upload" : "download";
}
void setupForS3();
private:
friend struct curlFileTransfer;
void setupForS3();
#if NIX_WITH_AWS_AUTH
std::optional<std::string> awsSigV4Provider;
#endif
@@ -228,11 +202,6 @@ struct FileTransfer
*/
FileTransferResult upload(const FileTransferRequest & request);
/**
* Synchronously delete a resource.
*/
FileTransferResult deleteResource(const FileTransferRequest & request);
/**
* Download a file, writing its data to a sink. The sink will be
* invoked on the thread of the caller.
+5 -5
View File
@@ -77,7 +77,7 @@ public:
Settings();
static unsigned int getDefaultCores();
unsigned int getDefaultCores() const;
Path nixPrefix;
@@ -427,7 +427,7 @@ public:
R"(
If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available.
It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
It means that remote build hosts fetches as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all.
This can drastically reduce build times if the network connection between the local machine and the remote build host is slow.
)"};
@@ -503,7 +503,7 @@ public:
by the Nix account, its group should be the group specified here,
and its mode should be `1775`.
If the build users group is empty, builds are performed under
If the build users group is empty, builds areperformed under
the uid of the Nix process (that is, the uid of the caller if
`NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
`NIX_REMOTE` is `daemon`). Obviously, this should not be used
@@ -847,8 +847,8 @@ public:
4. The path to the build's scratch directory. This directory
exists only if the build was run with `--keep-failed`.
The stderr and stdout output from the diff hook isn't displayed
to the user. Instead, it prints to the nix-daemon's log.
The stderr and stdout output from the diff hook isn't
displayed to the user. Instead, it print to the nix-daemon's log.
When using the Nix daemon, `diff-hook` must be set in the `nix.conf`
configuration file, and cannot be passed at the command line.
@@ -3,10 +3,6 @@
#include "nix/util/url.hh"
#include "nix/store/binary-cache-store.hh"
#include "nix/store/filetransfer.hh"
#include "nix/util/sync.hh"
#include <chrono>
namespace nix {
@@ -50,51 +46,4 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this<HttpBinaryCache
StoreReference getReference() const override;
};
class HttpBinaryCacheStore : public virtual BinaryCacheStore
{
struct State
{
bool enabled = true;
std::chrono::steady_clock::time_point disabledUntil;
};
Sync<State> _state;
public:
using Config = HttpBinaryCacheStoreConfig;
ref<Config> config;
HttpBinaryCacheStore(ref<Config> config);
void init() override;
protected:
std::optional<std::string> getCompressionMethod(const std::string & path);
void maybeDisable();
void checkEnabled();
bool fileExists(const std::string & path) override;
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
FileTransferRequest makeRequest(std::string_view path);
void getFile(const std::string & path, Sink & sink) override;
void getFile(const std::string & path, Callback<std::optional<std::string>> callback) noexcept override;
std::optional<std::string> getNixCacheInfo() override;
std::optional<TrustedFlag> isTrustedClient() override;
};
} // namespace nix
@@ -78,6 +78,7 @@ struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore
LocalFSStore(const Config & params);
void narFromPath(const StorePath & path, Sink & sink) override;
ref<SourceAccessor> getFSAccessor(bool requireValidPath = true) override;
std::shared_ptr<SourceAccessor> getFSAccessor(const StorePath & path, bool requireValidPath = true) override;
@@ -18,7 +18,7 @@ struct StructuredAttrs
{
static constexpr std::string_view envVarName{"__json"};
nlohmann::json::object_t structuredAttrs;
nlohmann::json structuredAttrs;
bool operator==(const StructuredAttrs &) const = default;
@@ -45,7 +45,7 @@ struct StructuredAttrs
*/
static void checkKeyNotInUse(const StringPairs & env);
nlohmann::json::object_t prepareStructuredAttrs(
nlohmann::json prepareStructuredAttrs(
Store & store,
const DerivationOptions & drvOptions,
const StorePathSet & inputPaths,
@@ -62,7 +62,7 @@ struct StructuredAttrs
* `prepareStructuredAttrs`, *not* the original `structuredAttrs`
* field.
*/
static std::string writeShell(const nlohmann::json::object_t & prepared);
static std::string writeShell(const nlohmann::json & prepared);
};
} // namespace nix
@@ -3,10 +3,6 @@
#include "nix/store/references.hh"
#include "nix/store/path.hh"
#include "nix/util/source-accessor.hh"
#include <functional>
#include <vector>
namespace nix {
@@ -25,57 +21,4 @@ public:
StorePathSet getResultPaths();
};
/**
* Result of scanning a single file for references.
*/
struct FileRefScanResult
{
CanonPath filePath; ///< The file that was scanned
StorePathSet foundRefs; ///< Which store paths were found in this file
};
/**
* Scan a store path tree and report which references appear in which files.
*
* This is like scanForReferences() but provides per-file granularity.
* Useful for cycle detection and detailed dependency analysis like `nix why-depends --precise`.
*
* The function walks the tree using the provided accessor and streams each file's
* contents through a RefScanSink to detect hash references. For each file that
* contains at least one reference, a callback is invoked with the file path and
* the set of references found.
*
* Note: This function only searches for the hash part of store paths (e.g.,
* "dc04vv14dak1c1r48qa0m23vr9jy8sm0"), not the name part. A store path like
* "/nix/store/dc04vv14dak1c1r48qa0m23vr9jy8sm0-foo" will be detected if the
* hash appears anywhere in the scanned content, regardless of the "-foo" suffix.
*
* @param accessor Source accessor to read the tree
* @param rootPath Root path to scan
* @param refs Set of store paths to search for
* @param callback Called for each file that contains at least one reference
*/
void scanForReferencesDeep(
SourceAccessor & accessor,
const CanonPath & rootPath,
const StorePathSet & refs,
std::function<void(FileRefScanResult)> callback);
/**
* Scan a store path tree and return which references appear in which files.
*
* This is a convenience wrapper around the callback-based scanForReferencesDeep()
* that collects all results into a map for efficient lookups.
*
* Note: This function only searches for the hash part of store paths, not the name part.
* See the callback-based overload for details.
*
* @param accessor Source accessor to read the tree
* @param rootPath Root path to scan
* @param refs Set of store paths to search for
* @return Map from file paths to the set of references found in each file
*/
std::map<CanonPath, StorePathSet>
scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs);
} // namespace nix
@@ -21,6 +21,8 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
Nix uses the `default` profile.
)"};
public:
const Setting<std::string> region{
this,
"us-east-1",
@@ -61,12 +63,6 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
> addressing instead of virtual host based addressing.
)"};
/**
* Set of settings that are part of the S3 URI itself.
* These are needed for region specification and other S3-specific settings.
*/
const std::set<const AbstractSetting *> s3UriSettings = {&profile, &region, &scheme, &endpoint};
static const std::string name()
{
return "S3 Binary Cache Store";
@@ -75,10 +71,6 @@ struct S3BinaryCacheStoreConfig : HttpBinaryCacheStoreConfig
static StringSet uriSchemes();
static std::string doc();
std::string getHumanReadableURI() const override;
ref<Store> openStore() const override;
};
} // namespace nix
-1
View File
@@ -26,7 +26,6 @@ struct ParsedS3URL
std::optional<std::string> profile;
std::optional<std::string> region;
std::optional<std::string> scheme;
std::optional<std::string> versionId;
/**
* The endpoint can be either missing, be an absolute URI (with a scheme like `http:`)
* or an authority (so an IP address or a registered name).
+1 -1
View File
@@ -609,7 +609,7 @@ public:
/**
* Write a NAR dump of a store path.
*/
virtual void narFromPath(const StorePath & path, Sink & sink);
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
/**
* For each path, if it's a derivation, build it. Building a
@@ -91,7 +91,7 @@ struct StoreDirConfig
std::pair<StorePath, Hash> computeStorePath(
std::string_view name,
const SourcePath & path,
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive,
ContentAddressMethod method = FileIngestionMethod::NixArchive,
HashAlgorithm hashAlgo = HashAlgorithm::SHA256,
const StorePathSet & references = {},
PathFilter & filter = defaultPathFilter) const;
@@ -68,7 +68,7 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore
void narFromPath(const StorePath & path, Sink & sink) override
{
Store::narFromPath(path, sink);
LocalFSStore::narFromPath(path, sink);
}
/**
@@ -13,7 +13,6 @@ namespace nix {
/* Note: you generally shouldn't change the protocol version. Define a
new `WorkerProto::Feature` instead. */
#define PROTOCOL_VERSION (1 << 8 | 38)
#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -153,7 +152,6 @@ enum struct WorkerProto::Op : uint64_t {
AddIndirectRoot = 12,
SyncWithGC = 13,
FindRoots = 14,
// ExportPath = 16, // removed
QueryDeriver = 18, // obsolete
SetOptions = 19,
CollectGarbage = 20,
@@ -163,7 +161,6 @@ enum struct WorkerProto::Op : uint64_t {
QueryFailedPaths = 24,
ClearFailedPaths = 25,
QueryPathInfo = 26,
// ImportPaths = 27, // removed
QueryDerivationOutputNames = 28, // obsolete
QueryPathFromHashPart = 29,
QuerySubstitutablePathInfos = 30,
+1 -2
View File
@@ -56,8 +56,7 @@ protected:
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override
const std::string & mimeType) override
{
auto path2 = config->binaryCacheDir + "/" + path;
static std::atomic<int> counter{0};
+7
View File
@@ -112,6 +112,13 @@ std::shared_ptr<SourceAccessor> LocalFSStore::getFSAccessor(const StorePath & pa
return std::make_shared<PosixSourceAccessor>(std::move(absPath));
}
void LocalFSStore::narFromPath(const StorePath & path, Sink & sink)
{
if (!isValidPath(path))
throw Error("path '%s' is not valid", printStorePath(path));
dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink);
}
const std::string LocalFSStore::drvsLogDir = "drvs";
std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path)
+4 -2
View File
@@ -1048,13 +1048,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
/* In case we are not interested in reading the NAR: discard it. */
bool narRead = false;
Finally cleanup = [&]() {
if (!narRead)
if (!narRead) {
NullFileSystemObjectSink sink;
try {
source.skip(info.narSize);
parseDump(sink, source);
} catch (...) {
// TODO: should Interrupted be handled here?
ignoreExceptionInDestructor();
}
}
};
addTempRoot(info.path);
+4 -6
View File
@@ -159,19 +159,17 @@ NarInfo NarInfo::fromJSON(const StoreDirConfig & store, const StorePath & path,
UnkeyedValidPathInfo::fromJSON(store, json),
}};
auto & obj = getObject(json);
if (json.contains("url"))
res.url = getString(valueAt(obj, "url"));
res.url = getString(valueAt(json, "url"));
if (json.contains("compression"))
res.compression = getString(valueAt(obj, "compression"));
res.compression = getString(valueAt(json, "compression"));
if (json.contains("downloadHash"))
res.fileHash = Hash::parseAny(getString(valueAt(obj, "downloadHash")), std::nullopt);
res.fileHash = Hash::parseAny(getString(valueAt(json, "downloadHash")), std::nullopt);
if (json.contains("downloadSize"))
res.fileSize = getUnsigned(valueAt(obj, "downloadSize"));
res.fileSize = getUnsigned(valueAt(json, "downloadSize"));
return res;
}
+1 -1
View File
@@ -312,7 +312,7 @@ void LocalStore::optimiseStore()
optimiseStore(stats);
printInfo("%s freed by hard-linking %d files", renderSize(stats.bytesFreed), stats.filesLinked);
printInfo("%s freed by hard-linking %d files", showBytes(stats.bytesFreed), stats.filesLinked);
}
void LocalStore::optimisePath(const Path & path, RepairFlag repair)
+4 -5
View File
@@ -33,8 +33,7 @@ std::optional<StructuredAttrs> StructuredAttrs::tryExtract(StringPairs & env)
std::pair<std::string_view, std::string> StructuredAttrs::unparse() const
{
// TODO don't copy the JSON object just to dump it.
return {envVarName, static_cast<nlohmann::json>(structuredAttrs).dump()};
return {envVarName, structuredAttrs.dump()};
}
void StructuredAttrs::checkKeyNotInUse(const StringPairs & env)
@@ -98,7 +97,7 @@ static nlohmann::json pathInfoToJSON(Store & store, const StorePathSet & storePa
return jsonList;
}
nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
nlohmann::json StructuredAttrs::prepareStructuredAttrs(
Store & store,
const DerivationOptions & drvOptions,
const StorePathSet & inputPaths,
@@ -121,7 +120,7 @@ nlohmann::json::object_t StructuredAttrs::prepareStructuredAttrs(
return json;
}
std::string StructuredAttrs::writeShell(const nlohmann::json::object_t & json)
std::string StructuredAttrs::writeShell(const nlohmann::json & json)
{
auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
@@ -145,7 +144,7 @@ std::string StructuredAttrs::writeShell(const nlohmann::json::object_t & json)
std::string jsonSh;
for (auto & [key, value] : json) {
for (auto & [key, value] : json.items()) {
if (!std::regex_match(key, shVarName))
continue;
-90
View File
@@ -1,15 +1,11 @@
#include "nix/store/path-references.hh"
#include "nix/util/hash.hh"
#include "nix/util/archive.hh"
#include "nix/util/source-accessor.hh"
#include "nix/util/canon-path.hh"
#include "nix/util/logging.hh"
#include <map>
#include <cstdlib>
#include <mutex>
#include <algorithm>
#include <functional>
namespace nix {
@@ -58,90 +54,4 @@ StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathS
return refsSink.getResultPaths();
}
void scanForReferencesDeep(
SourceAccessor & accessor,
const CanonPath & rootPath,
const StorePathSet & refs,
std::function<void(FileRefScanResult)> callback)
{
// Recursive tree walker
auto walk = [&](this auto & self, const CanonPath & path) -> void {
auto stat = accessor.lstat(path);
switch (stat.type) {
case SourceAccessor::tRegular: {
// Create a fresh sink for each file to independently detect references.
// RefScanSink accumulates found hashes globally - once a hash is found,
// it remains in the result set. If we reused the same sink across files,
// we couldn't distinguish which files contain which references, as a hash
// found in an earlier file wouldn't be reported when found in later files.
PathRefScanSink sink = PathRefScanSink::fromPaths(refs);
// Scan this file by streaming its contents through the sink
accessor.readFile(path, sink);
// Get the references found in this file
auto foundRefs = sink.getResultPaths();
// Report if we found anything in this file
if (!foundRefs.empty()) {
debug("scanForReferencesDeep: found %d references in %s", foundRefs.size(), path.abs());
callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)});
}
break;
}
case SourceAccessor::tDirectory: {
// Recursively scan directory contents
auto entries = accessor.readDirectory(path);
for (const auto & [name, entryType] : entries) {
self(path / name);
}
break;
}
case SourceAccessor::tSymlink: {
// Create a fresh sink for the symlink target (same reason as regular files)
PathRefScanSink sink = PathRefScanSink::fromPaths(refs);
// Scan symlink target for references
auto target = accessor.readLink(path);
sink(std::string_view(target));
// Get the references found in this symlink target
auto foundRefs = sink.getResultPaths();
if (!foundRefs.empty()) {
debug("scanForReferencesDeep: found %d references in symlink %s", foundRefs.size(), path.abs());
callback(FileRefScanResult{.filePath = path, .foundRefs = std::move(foundRefs)});
}
break;
}
case SourceAccessor::tChar:
case SourceAccessor::tBlock:
case SourceAccessor::tSocket:
case SourceAccessor::tFifo:
case SourceAccessor::tUnknown:
default:
throw Error("file '%s' has an unsupported type", path.abs());
}
};
// Start the recursive walk from the root
walk(rootPath);
}
std::map<CanonPath, StorePathSet>
scanForReferencesDeep(SourceAccessor & accessor, const CanonPath & rootPath, const StorePathSet & refs)
{
std::map<CanonPath, StorePathSet> results;
scanForReferencesDeep(accessor, rootPath, refs, [&](FileRefScanResult result) {
results[std::move(result.filePath)] = std::move(result.foundRefs);
});
return results;
}
} // namespace nix
+1 -1
View File
@@ -73,7 +73,7 @@ void RemoteStore::initConnection(Connection & conn)
try {
auto [protoVersion, features] =
WorkerProto::BasicClientConnection::handshake(conn.to, tee, PROTOCOL_VERSION, WorkerProto::allFeatures);
if (protoVersion < MINIMUM_PROTOCOL_VERSION)
if (protoVersion < 256 + 18)
throw Error("the Nix daemon version is too old");
conn.protoVersion = protoVersion;
conn.features = features;
+1 -1
View File
@@ -226,7 +226,7 @@ void RestrictedStore::narFromPath(const StorePath & path, Sink & sink)
{
if (!goal.isAllowed(path))
throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
Store::narFromPath(path, sink);
LocalFSStore::narFromPath(path, sink);
}
void RestrictedStore::ensurePath(const StorePath & path)
+8 -153
View File
@@ -1,138 +1,12 @@
#include "nix/store/s3-binary-cache-store.hh"
#include <cassert>
#include "nix/store/http-binary-cache-store.hh"
#include "nix/store/store-registration.hh"
#include <cassert>
#include <ranges>
#include <regex>
namespace nix {
class S3BinaryCacheStore : public virtual HttpBinaryCacheStore
{
public:
S3BinaryCacheStore(ref<S3BinaryCacheStoreConfig> config)
: Store{*config}
, BinaryCacheStore{*config}
, HttpBinaryCacheStore{config}
, s3Config{config}
{
}
void upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint) override;
private:
ref<S3BinaryCacheStoreConfig> s3Config;
/**
* Creates a multipart upload for large objects to S3.
*
* @see
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html#API_CreateMultipartUpload_RequestSyntax
*/
std::string createMultipartUpload(
std::string_view key, std::string_view mimeType, std::optional<std::string_view> contentEncoding);
/**
* Uploads a single part of a multipart upload
*
* @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html#API_UploadPart_RequestSyntax
*
* @returns the [ETag](https://en.wikipedia.org/wiki/HTTP_ETag)
*/
std::string uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data);
/**
* Abort a multipart upload
*
* @see
* https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html#API_AbortMultipartUpload_RequestSyntax
*/
void abortMultipartUpload(std::string_view key, std::string_view uploadId);
};
void S3BinaryCacheStore::upsertFile(
const std::string & path,
std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
uint64_t sizeHint)
{
HttpBinaryCacheStore::upsertFile(path, istream, mimeType, sizeHint);
}
std::string S3BinaryCacheStore::createMultipartUpload(
std::string_view key, std::string_view mimeType, std::optional<std::string_view> contentEncoding)
{
auto req = makeRequest(key);
// setupForS3() converts s3:// to https:// but strips query parameters
// So we call it first, then add our multipart parameters
req.setupForS3();
auto url = req.uri.parsed();
url.query["uploads"] = "";
req.uri = VerbatimURL(url);
req.method = HttpMethod::POST;
req.data = "";
req.mimeType = mimeType;
if (contentEncoding) {
req.headers.emplace_back("Content-Encoding", *contentEncoding);
}
auto result = getFileTransfer()->enqueueFileTransfer(req).get();
std::regex uploadIdRegex("<UploadId>([^<]+)</UploadId>");
std::smatch match;
if (std::regex_search(result.data, match, uploadIdRegex)) {
return match[1];
}
throw Error("S3 CreateMultipartUpload response missing <UploadId>");
}
std::string
S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId, uint64_t partNumber, std::string data)
{
auto req = makeRequest(key);
req.method = HttpMethod::PUT;
req.setupForS3();
auto url = req.uri.parsed();
url.query["partNumber"] = std::to_string(partNumber);
url.query["uploadId"] = uploadId;
req.uri = VerbatimURL(url);
req.data = std::move(data);
req.mimeType = "application/octet-stream";
auto result = getFileTransfer()->enqueueFileTransfer(req).get();
if (result.etag.empty()) {
throw Error("S3 UploadPart response missing ETag for part %d", partNumber);
}
return std::move(result.etag);
}
void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_view uploadId)
{
auto req = makeRequest(key);
req.setupForS3();
auto url = req.uri.parsed();
url.query["uploadId"] = uploadId;
req.uri = VerbatimURL(url);
req.method = HttpMethod::DELETE;
getFileTransfer()->enqueueFileTransfer(req).get();
}
StringSet S3BinaryCacheStoreConfig::uriSchemes()
{
return {"s3"};
@@ -143,31 +17,19 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig(
: StoreConfig(params)
, HttpBinaryCacheStoreConfig(scheme, _cacheUri, params)
{
// For S3 stores, preserve S3-specific query parameters as part of the URL
// These are needed for region specification and other S3-specific settings
assert(cacheUri.query.empty());
assert(cacheUri.scheme == "s3");
// Only copy S3-specific parameters to the URL query
static const std::set<std::string> s3Params = {"region", "endpoint", "profile", "scheme"};
for (const auto & [key, value] : params) {
auto s3Params =
std::views::transform(s3UriSettings, [](const AbstractSetting * setting) { return setting->name; });
if (std::ranges::contains(s3Params, key)) {
if (s3Params.contains(key)) {
cacheUri.query[key] = value;
}
}
}
std::string S3BinaryCacheStoreConfig::getHumanReadableURI() const
{
auto reference = getReference();
reference.params = [&]() {
Params relevantParams;
for (auto & setting : s3UriSettings)
if (setting->overridden)
relevantParams.insert({setting->name, reference.params.at(setting->name)});
return relevantParams;
}();
return reference.render();
}
std::string S3BinaryCacheStoreConfig::doc()
{
return R"(
@@ -177,13 +39,6 @@ std::string S3BinaryCacheStoreConfig::doc()
)";
}
ref<Store> S3BinaryCacheStoreConfig::openStore() const
{
auto sharedThis = std::const_pointer_cast<S3BinaryCacheStoreConfig>(
std::static_pointer_cast<const S3BinaryCacheStoreConfig>(shared_from_this()));
return make_ref<S3BinaryCacheStore>(ref{sharedThis});
}
static RegisterStoreImplementation<S3BinaryCacheStoreConfig> registerS3BinaryCacheStore;
} // namespace nix
+2 -3
View File
@@ -27,8 +27,7 @@ like the following to be accessible:
"Sid": "AllowDirectReads",
"Action": [
"s3:GetObject",
"s3:GetBucketLocation",
"s3:ListBucket"
"s3:GetBucketLocation"
],
"Effect": "Allow",
"Resource": [
@@ -52,7 +51,7 @@ Consult the documentation linked above for further details.
### Authenticated reads to your S3 binary cache
Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject`, `s3:GetBucketLocation`, and `s3:ListBucket` actions on all objects in the bucket.
Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket.
The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this.
### Authenticated writes to your S3-compatible binary cache
-10
View File
@@ -48,7 +48,6 @@ try {
.profile = getOptionalParam("profile"),
.region = getOptionalParam("region"),
.scheme = getOptionalParam("scheme"),
.versionId = getOptionalParam("versionId"),
.endpoint = [&]() -> decltype(ParsedS3URL::endpoint) {
if (!endpoint)
return std::monostate();
@@ -74,12 +73,6 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
auto regionStr = region.transform(toView).value_or("us-east-1");
auto schemeStr = scheme.transform(toView).value_or("https");
// Build query parameters (e.g., versionId if present)
StringMap queryParams;
if (versionId) {
queryParams["versionId"] = *versionId;
}
// Handle endpoint configuration using std::visit
return std::visit(
overloaded{
@@ -92,7 +85,6 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
.scheme = std::string{schemeStr},
.authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"},
.path = std::move(path),
.query = std::move(queryParams),
};
},
[&](const ParsedURL::Authority & auth) {
@@ -104,7 +96,6 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
.scheme = std::string{schemeStr},
.authority = auth,
.path = std::move(path),
.query = std::move(queryParams),
};
},
[&](const ParsedURL & endpointUrl) {
@@ -116,7 +107,6 @@ ParsedURL ParsedS3URL::toHttpsUrl() const
.scheme = endpointUrl.scheme,
.authority = endpointUrl.authority,
.path = std::move(path),
.query = std::move(queryParams),
};
},
},
+1 -1
View File
@@ -143,7 +143,7 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore
void narFromPath(const StorePath & path, Sink & sink) override
{
return Store::narFromPath(path, sink);
return LocalFSStore::narFromPath(path, sink);
}
ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
-7
View File
@@ -300,13 +300,6 @@ ValidPathInfo Store::addToStoreSlow(
return info;
}
void Store::narFromPath(const StorePath & path, Sink & sink)
{
auto accessor = requireStoreObjectAccessor(path);
SourcePath sourcePath{accessor};
dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive);
}
StringSet Store::Config::getDefaultSystemFeatures()
{
auto res = settings.systemFeatures.get();
Loaded 100 of 136 files, more files were not shown because too many files have changed in this diff. Show more