Compare commits

..
Author SHA1 Message Date
John Ericson 156061409a Do some parsing of git hashes based on length.
Note that for Nix-native information we should *not* do length tricks,
but instead always rely on an explicit algorithm. This hack should be
only for foreign hash literals.
2025-11-09 18:52:35 -05:00
John Ericson 6da56d7ed4 Make changes to support libgit2 experimental API
This experimental interface is likely to become stable with libgit2 2.0.
2025-11-09 18:52:35 -05:00
John Ericson 9846305fee Switch to libgit2 experimental
Just patch the package for now, but we should make Nixpkgs provide what
we need (whether that the experimental variant, or a 2.x release with
the new API made non-experimental) before releasing this change.
2025-11-09 18:52:35 -05:00
303 changed files with 2219 additions and 6799 deletions

No files matched your search

+1 -1
View File
@@ -174,7 +174,7 @@ jobs:
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)" TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT" echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4 - uses: cachix/install-nix-action@456688f15bc354bef6d396e4a35f4f89d40bf2b7 # v31.8.2
if: ${{ !matrix.experimental-installer }} if: ${{ !matrix.experimental-installer }}
with: with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
+6 -3
View File
@@ -37,7 +37,6 @@ mkMesonDerivation (finalAttrs: {
(fileset.unions [ (fileset.unions [
../../.version ../../.version
# For example JSON # For example JSON
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash ../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/store-path
@@ -59,8 +58,8 @@ mkMesonDerivation (finalAttrs: {
"man" "man"
]; ];
nativeBuildInputs = [ # Hack for sake of the dev shell
nix-cli passthru.externalNativeBuildInputs = [
meson meson
ninja ninja
(lib.getBin lowdown-unsandboxed) (lib.getBin lowdown-unsandboxed)
@@ -79,6 +78,10 @@ mkMesonDerivation (finalAttrs: {
changelog-d changelog-d
]; ];
nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [
nix-cli
];
preConfigure = '' preConfigure = ''
chmod u+w ./.version chmod u+w ./.version
echo ${finalAttrs.version} > ./.version echo ${finalAttrs.version} > ./.version
+1 -9
View File
@@ -22,15 +22,7 @@ The store path info JSON format has been updated from version 1 to version 2:
- New: `"ca": {"method": "nar", "hash": {"algorithm": "sha256", "format": "base64", "hash": "EMIJ+giQ..."}}` - New: `"ca": {"method": "nar", "hash": {"algorithm": "sha256", "format": "base64", "hash": "EMIJ+giQ..."}}`
- Still `null` values for input-addressed store objects - Still `null` values for input-addressed store objects
- **Structured hash fields**: Version 1 format is still accepted when reading for backward compatibility.
Hash values (`narHash` and `downloadHash`) are now structured JSON objects instead of strings:
- Old: `"narHash": "sha256:FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="`
- New: `"narHash": {"algorithm": "sha256", "format": "base64", "hash": "FePFYIlM..."}`
- Same structure applies to `downloadHash` in NAR info contexts
Nix currently only produces, and doesn't consume this format.
**Affected command**: `nix path-info --json` **Affected command**: `nix path-info --json`
-12
View File
@@ -1,12 +0,0 @@
---
synopsis: Fix "download buffer is full; consider increasing the 'download-buffer-size' setting" warning
prs: [14614]
issues: [11728]
---
The underlying issue that led to [#11728](https://github.com/NixOS/nix/issues/11728) has been resolved by utilizing
[libcurl write pausing functionality](https://curl.se/libcurl/c/curl_easy_pause.html) to control backpressure when unpacking to slow destinations like the git-backed tarball cache. The default value of `download-buffer-size` is now 1 MiB and it's no longer recommended to increase it, since the root cause has been fixed.
This is expected to improve download performance on fast connections, since previously a single slow download consumer would stall the thread and prevent any other transfers from progressing.
Many thanks go out to the [Lix project](https://lix.systems/) for the [implementation](https://git.lix.systems/lix-project/lix/commit/4ae6fb5a8f0d456b8d2ba2aaca3712b4e49057fc) that served as inspiration for this change and for triaging libcurl [issues with pausing](https://github.com/curl/curl/issues/19334).
-21
View File
@@ -1,21 +0,0 @@
---
synopsis: "S3 binary cache stores now support storage class configuration"
prs: [14464]
issues: [7015]
---
S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
Example usage:
```bash
# Use Glacier storage for long-term archival
nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
# Use Intelligent Tiering for automatic cost optimization
nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
```
The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.
-2
View File
@@ -29,7 +29,6 @@
- [Build Trace](store/build-trace.md) - [Build Trace](store/build-trace.md)
- [Derivation Resolution](store/resolution.md) - [Derivation Resolution](store/resolution.md)
- [Building](store/building.md) - [Building](store/building.md)
- [Secrets](store/secrets.md)
- [Store Types](store/types/index.md) - [Store Types](store/types/index.md)
{{#include ./store/types/SUMMARY.md}} {{#include ./store/types/SUMMARY.md}}
- [Appendix: Math notation](store/math-notation.md) - [Appendix: Math notation](store/math-notation.md)
@@ -121,7 +120,6 @@
- [Architecture and Design](architecture/architecture.md) - [Architecture and Design](architecture/architecture.md)
- [Formats and Protocols](protocols/index.md) - [Formats and Protocols](protocols/index.md)
- [JSON Formats](protocols/json/index.md) - [JSON Formats](protocols/json/index.md)
- [File System Object](protocols/json/file-system-object.md)
- [Hash](protocols/json/hash.md) - [Hash](protocols/json/hash.md)
- [Content Address](protocols/json/content-address.md) - [Content Address](protocols/json/content-address.md)
- [Store Path](protocols/json/store-path.md) - [Store Path](protocols/json/store-path.md)
+3 -3
View File
@@ -36,7 +36,7 @@ to a temporary location. The tarball must include a single top-level
directory containing at least a file named `default.nix`. directory containing at least a file named `default.nix`.
`nix-build` is essentially a wrapper around `nix-build` is essentially a wrapper around
[`nix-instantiate`](./nix-instantiate.md) (to translate a high-level Nix [`nix-instantiate`](nix-instantiate.md) (to translate a high-level Nix
expression to a low-level [store derivation]) and [`nix-store expression to a low-level [store derivation]) and [`nix-store
--realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
derivation). derivation).
@@ -52,8 +52,8 @@ derivation).
# Options # Options
All options not listed here are passed to All options not listed here are passed to
[`nix-store --realise`](./nix-store/realise.md), [`nix-store --realise`](nix-store/realise.md),
except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](./nix-instantiate.md). except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md).
- <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span> - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span>
@@ -32,7 +32,7 @@ standard input.
- `--add-root` *path* - `--add-root` *path*
See the [corresponding option](./nix-store.md) in `nix-store`. See the [corresponding option](nix-store.md) in `nix-store`.
- `--parse` - `--parse`
+1 -1
View File
@@ -15,7 +15,7 @@ In the development shell, set the `mesonBuildType` environment variable to `debu
Then, proceed to build Nix as described in [Building Nix](./building.md). Then, proceed to build Nix as described in [Building Nix](./building.md).
This will build Nix with debug symbols, which are essential for effective debugging. This will build Nix with debug symbols, which are essential for effective debugging.
It is also possible to build without optimization for faster build: It is also possible to build without debugging for faster build:
```console ```console
[nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify) [nix-shell]$ NIX_HARDENING_ENABLE=$(printLines $NIX_HARDENING_ENABLE | grep -v fortify)
@@ -1,21 +0,0 @@
{{#include file-system-object-v1-fixed.md}}
## Examples
### Simple
```json
{{#include schema/file-system-object-v1/simple.json}}
```
### Complex
```json
{{#include schema/file-system-object-v1/complex.json}}
```
<!-- need to convert YAML to JSON first
## Raw Schema
[JSON Schema for File System Object v1](schema/file-system-object-v1.json)
-->
@@ -11,7 +11,6 @@ s/\\`/`/g
# #
# As we have more such relative links, more replacements of this nature # As we have more such relative links, more replacements of this nature
# should appear below. # should appear below.
s^#/\$defs/\(regular\|symlink\|directory\)^In this schema^g
s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g s^\(./hash-v1.yaml\)\?#/$defs/algorithm^[JSON format for `Hash`](./hash.html#algorithm)^g
s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g s^\(./hash-v1.yaml\)^[JSON format for `Hash`](./hash.html)^g
s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g s^\(./content-address-v1.yaml\)\?#/$defs/method^[JSON format for `ContentAddress`](./content-address.html#method)^g
@@ -9,7 +9,6 @@ json_schema_for_humans = find_program('generate-schema-doc', required : false)
json_schema_config = files('json-schema-for-humans-config.yaml') json_schema_config = files('json-schema-for-humans-config.yaml')
schemas = [ schemas = [
'file-system-object-v1',
'hash-v1', 'hash-v1',
'content-address-v1', 'content-address-v1',
'store-path-v1', 'store-path-v1',
@@ -1 +0,0 @@
../../../../../../src/libutil-tests/data/memory-source-accessor
@@ -1,71 +0,0 @@
"$schema": http://json-schema.org/draft-04/schema#
"$id": https://nix.dev/manual/nix/latest/protocols/json/schema/file-system-object-v1.json
title: File System Object
description: |
This schema describes the JSON representation of Nix's [File System Object](@docroot@/store/file-system-object.md).
The schema is recursive because file system objects contain other file system objects.
type: object
required: ["type"]
properties:
type:
type: string
enum: ["regular", "symlink", "directory"]
# Enforce conditional structure based on `type`
anyOf:
- $ref: "#/$defs/regular"
required: ["type", "contents"]
- $ref: "#/$defs/directory"
required: ["type", "entries"]
- $ref: "#/$defs/symlink"
required: ["type", "target"]
"$defs":
regular:
title: Regular File
description: |
See [Regular File](@docroot@/store/file-system-object.md#regular) in the manual for details.
required: ["contents"]
properties:
type:
const: "regular"
contents:
type: string
description: File contents
executable:
type: boolean
description: Whether the file is executable.
default: false
additionalProperties: false
directory:
title: Directory
description: |
See [Directory](@docroot@/store/file-system-object.md#directory) in the manual for details.
required: ["entries"]
properties:
type:
const: "directory"
entries:
type: object
description: |
Map of names to nested file system objects (for type=directory)
additionalProperties:
$ref: "#"
additionalProperties: false
symlink:
title: Symbolic Link
description: |
See [Symbolic Link](@docroot@/store/file-system-object.md#symlink) in the manual for details.
required: ["target"]
properties:
type:
const: "symlink"
target:
type: string
description: Target path of the symlink.
additionalProperties: false
@@ -71,7 +71,7 @@ $defs:
Note: This field may not be present in all contexts, such as when the path is used as the key and the the store object info the value in map. Note: This field may not be present in all contexts, such as when the path is used as the key and the the store object info the value in map.
narHash: narHash:
"$ref": "./hash-v1.yaml" type: string
title: NAR Hash title: NAR Hash
description: | description: |
Hash of the [file system object](@docroot@/store/file-system-object.md) part of the store object when serialized as a [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive). Hash of the [file system object](@docroot@/store/file-system-object.md) part of the store object when serialized as a [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive).
@@ -229,7 +229,7 @@ $defs:
> This is an impure "`.narinfo`" field that may not be included in certain contexts. > This is an impure "`.narinfo`" field that may not be included in certain contexts.
downloadHash: downloadHash:
"$ref": "./hash-v1.yaml" type: string
title: Download Hash title: Download Hash
description: | description: |
A digest for the compressed archive itself, as opposed to the data contained within. A digest for the compressed archive itself, as opposed to the data contained within.
@@ -24,7 +24,7 @@ nar-obj-inner
| str("type"), str("directory") directory | str("type"), str("directory") directory
; ;
regular = [ str("executable"), str("") ], str("contents"), str(contents); regular = [ str("executable") ], str("contents"), str(contents);
symlink = str("target"), str(target); symlink = str("target"), str(target);
@@ -52,4 +52,4 @@ The Nix Archive (NAR) format is also formally described using [Kaitai Struct](ht
{{#include nar.ksy}} {{#include nar.ksy}}
``` ```
The source of the spec can be found [here](https://github.com/nixos/nix/blob/master/src/nix-manual/source/protocols/nix-archive/nar.ksy). Contributions and improvements to the spec are welcomed. The source of the spec can be found [here](https://github.com/nixos/nix/blob/master/src/nix-manual/source/protocols/nix-archive/nar.ksy). Contributions and improvements to the spec are welcomed.
@@ -3,23 +3,19 @@
Nix uses a simplified model of the file system, which consists of file system objects. Nix uses a simplified model of the file system, which consists of file system objects.
Every file system object is one of the following: Every file system object is one of the following:
- [**Regular File**]{#regular} - File
- A possibly empty sequence of bytes for contents - A possibly empty sequence of bytes for contents
- A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission
- [**Directory**]{#directory} - Directory
Mapping of names to child file system objects Mapping of names to child file system objects
- [**Symbolic link**]{#symlink} - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link)
An arbitrary string, known as the *target* of the symlink. An arbitrary string.
Nix does not assign any semantics to symbolic links.
In general, Nix does not assign any semantics to symbolic links.
Certain operations however, may make additional assumptions and attempt to use the target to find another file system object.
> See [the Wikpedia article on symbolic links](https://en.m.wikipedia.org/wiki/Symbolic_link) for background information if you are unfamiliar with this Unix concept.
File system objects and their children form a tree. File system objects and their children form a tree.
A bare file or symlink can be a root file system object. A bare file or symlink can be a root file system object.
-20
View File
@@ -1,20 +0,0 @@
# Secrets
The store is readable to all users on the system. For this reason, it
is generally discouraged to allow secrets to make it into the store.
Even on a single-user system, separate system users isolate services
from each other and having secrets that all local users can read
weakens that isolation. When using external store caches the secrets
may end up there, and on multi-user systems the secrets will be
available to all those users.
Organize your derivations so that secrets are read from the filesystem
(with appropriate access controls) at run time. Place the secrets on
the filesystem manually or use a scheme that includes the secret in
the store in encrypted form, and decrypts it adding the relevant
access control on system activation.
Several such schemes for NixOS can in the
[comparison of secret managing schemes] on the wiki.
[comparison of secret managing schemes]: https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes
+1 -6
View File
@@ -60,9 +60,4 @@ if get_option('unit-tests')
subproject('libflake-tests') subproject('libflake-tests')
endif endif
subproject('nix-functional-tests') subproject('nix-functional-tests')
if get_option('json-schema-checks') subproject('json-schema-checks')
subproject('json-schema-checks')
endif
if get_option('kaitai-struct-checks')
subproject('kaitai-struct-checks')
endif
-14
View File
@@ -27,17 +27,3 @@ option(
value : false, value : false,
description : 'Build benchmarks (requires gbenchmark)', description : 'Build benchmarks (requires gbenchmark)',
) )
option(
'kaitai-struct-checks',
type : 'boolean',
value : true,
description : 'Check the Kaitai Struct specifications (requires Kaitai Struct)',
)
option(
'json-schema-checks',
type : 'boolean',
value : true,
description : 'Check JSON schema validity of schemas and examples (requires jv)',
)
+10
View File
@@ -74,4 +74,14 @@ scope: {
buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase;
installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase;
}); });
libgit2 = pkgs.libgit2.overrideAttrs (attrs: {
cmakeFlags = (attrs.cmakeFlags or [ ]) ++ [
(lib.mesonBool "EXPERIMENTAL_SHA256" true)
];
postInstall = (attrs.postInstall or "") + ''
substituteInPlace $(find $dev/include -type f) --replace-quiet '#include "git2/' '#include "git2-experimental/'
'';
});
} }
+25 -209
View File
@@ -3,118 +3,10 @@
devFlake, devFlake,
}: }:
let
# Some helper functions
/**
Compute a filtered closure of build inputs.
Specifically, `buildInputsClosure cond startSet` computes the closure formed
by recursive application of `p: filter cond p.buildInputs ++ filter cond p.propagatedBuildInputs`
to `startSet`.
Example:
```nix
builtInputsClosure isInternal [ pkg1 pkg2 ]
=> [ pkg1 pkg3 pkg2 pkg10 ]
```
Note: order tbd
Note: `startSet` is *NOT* filtered.
*/
buildInputsClosureCond =
cond: startSet:
let
closure = builtins.genericClosure {
startSet = map (d: {
key = d.drvPath;
value = d;
}) startSet;
operator =
d:
let
r =
map
(d': {
key = d'.drvPath;
value = d';
})
(
lib.filter cond d.value.buildInputs or [ ] ++ lib.filter cond d.value.propagatedBuildInputs or [ ]
);
in
r;
};
in
map (item: item.value) closure;
/**
`[ pkg1 pkg2 ]` -> `{ "...-pkg2.drv" = null; "...-pkg1.drv" = null }`
Note: fairly arbitrary order (hash based). Use for efficient set membership test only.
*/
byDrvPath =
l:
lib.listToAttrs (
map (c: {
name =
# Just a lookup key
builtins.unsafeDiscardStringContext c.drvPath;
value = null;
}) l
);
/**
Stable dedup.
Unlike `listToAttrs` -> `attrValues`, this preserves the input ordering,
which is more predictable ("deterministic") than e.g. sorting store paths,
whose hashes affect the ordering on every change.
*/
# TODO: add to Nixpkgs lib, refer from uniqueStrings
dedupByString =
key: l:
let
r =
lib.foldl'
(
a@{ list, set }:
elem:
let
k = builtins.unsafeDiscardStringContext (key elem);
in
if set ? ${k} then
a
else
let
# Note: O(n²) copying. Use linkedLists to concat them in one go at the end.
# https://github.com/NixOS/nixpkgs/pull/452088
newList = [ elem ] ++ list;
newSet = set // {
${k} = null;
};
in
builtins.seq newList builtins.seq newSet {
list = newList;
set = newSet;
}
)
{
list = [ ];
set = { };
}
l;
in
r.list;
in
{ pkgs }: { pkgs }:
# TODO: don't use nix-util for this?
pkgs.nixComponents2.nix-util.overrideAttrs ( pkgs.nixComponents2.nix-util.overrideAttrs (
finalAttrs: prevAttrs: attrs:
let let
stdenv = pkgs.nixDependencies2.stdenv; stdenv = pkgs.nixDependencies2.stdenv;
@@ -129,93 +21,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
"-D${prefix}:${rest}"; "-D${prefix}:${rest}";
havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
availableComponents = lib.filterAttrs (
k: v: lib.meta.availableOn pkgs.hostPlatform v
) allComponents;
activeComponents = buildInputsClosureCond isInternal (
lib.attrValues (finalAttrs.passthru.config.getComponents availableComponents)
);
allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
internalDrvs = byDrvPath (
# Drop the attr names (not present in buildInputs anyway)
lib.attrValues availableComponents
++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues availableComponents)
);
isInternal =
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
in in
{ {
pname = "shell-for-nix"; pname = "shell-for-" + attrs.pname;
passthru = {
inherit activeComponents;
# We use this attribute to store non-derivation values like functions and
# perhaps other things that are primarily for overriding and not the shell.
config = {
# Default getComponents
getComponents =
c:
builtins.removeAttrs c (
lib.optionals (!havePerl) [ "nix-perl-bindings" ]
++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]
);
};
/**
Produce a devShell for a given set of nix components
Example:
```nix
shell.withActiveComponents (c: {
inherit (c) nix-util;
})
```
*/
withActiveComponents =
f2:
finalAttrs.finalPackage.overrideAttrs (
finalAttrs: prevAttrs: {
passthru = prevAttrs.passthru // {
config = prevAttrs.passthru.config // {
getComponents = f2;
};
};
}
);
small =
(finalAttrs.finalPackage.withActiveComponents (
c:
lib.intersectAttrs (lib.genAttrs [
"nix-cli"
"nix-util-tests"
"nix-store-tests"
"nix-expr-tests"
"nix-fetchers-tests"
"nix-flake-tests"
"nix-functional-tests"
"nix-perl-bindings"
] (_: null)) c
)).overrideAttrs
(o: {
mesonFlags = o.mesonFlags ++ [
# TODO: infer from activeComponents or vice versa
"-Dkaitai-struct-checks=false"
"-Djson-schema-checks=false"
];
});
};
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
version = lib.fileContents ../.version; version = lib.fileContents ../.version;
name = finalAttrs.pname; name = attrs.pname;
installFlags = "sysconfdir=$(out)/etc"; installFlags = "sysconfdir=$(out)/etc";
shellHook = '' shellHook = ''
@@ -273,8 +85,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
CXX_LD = "mold"; CXX_LD = "mold";
}; };
dontUseCmakeConfigure = true;
mesonFlags = mesonFlags =
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags) map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags) ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
@@ -288,9 +98,17 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
nativeBuildInputs = nativeBuildInputs =
let let
inputs = inputs =
dedupByString (v: "${v}") ( attrs.nativeBuildInputs or [ ]
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents) ++ pkgs.nixComponents2.nix-util.nativeBuildInputs
) ++ pkgs.nixComponents2.nix-store.nativeBuildInputs
++ pkgs.nixComponents2.nix-fetchers.nativeBuildInputs
++ pkgs.nixComponents2.nix-expr.nativeBuildInputs
++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.nativeBuildInputs
++ lib.optionals buildCanExecuteHost pkgs.nixComponents2.nix-manual.externalNativeBuildInputs
++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs
++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs
++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs
++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs
++ lib.optional ( ++ lib.optional (
!buildCanExecuteHost !buildCanExecuteHost
# Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
@@ -299,7 +117,9 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
&& lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages) && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)
) pkgs.buildPackages.mesonEmulatorHook ) pkgs.buildPackages.mesonEmulatorHook
++ [ ++ [
pkgs.buildPackages.cmake
pkgs.buildPackages.gnused pkgs.buildPackages.gnused
pkgs.buildPackages.changelog-d
modular.pre-commit.settings.package modular.pre-commit.settings.package
(pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
pkgs.buildPackages.nixfmt-rfc-style pkgs.buildPackages.nixfmt-rfc-style
@@ -316,22 +136,18 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
# from making its way into NIX_CFLAGS_COMPILE. # from making its way into NIX_CFLAGS_COMPILE.
lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs; lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs;
propagatedNativeBuildInputs = dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (
lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents
)
);
buildInputs = [ buildInputs = [
pkgs.gbenchmark pkgs.gbenchmark
] ]
++ dedupByString (v: "${v}") ( ++ attrs.buildInputs or [ ]
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents) ++ pkgs.nixComponents2.nix-util.buildInputs
) ++ pkgs.nixComponents2.nix-store.buildInputs
++ pkgs.nixComponents2.nix-store-tests.externalBuildInputs
++ pkgs.nixComponents2.nix-fetchers.buildInputs
++ pkgs.nixComponents2.nix-expr.buildInputs
++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs
++ pkgs.nixComponents2.nix-cmd.buildInputs
++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs
++ lib.optional havePerl pkgs.perl; ++ lib.optional havePerl pkgs.perl;
propagatedBuildInputs = dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
);
} }
) )
@@ -1 +0,0 @@
../../src/libutil-tests/data/memory-source-accessor
-8
View File
@@ -20,14 +20,6 @@ schema_dir = meson.current_source_dir() / 'schema'
# Get all example files # Get all example files
schemas = [ schemas = [
{
'stem' : 'file-system-object',
'schema' : schema_dir / 'file-system-object-v1.yaml',
'files' : [
'simple.json',
'complex.json',
],
},
{ {
'stem' : 'hash', 'stem' : 'hash',
'schema' : schema_dir / 'hash-v1.yaml', 'schema' : schema_dir / 'hash-v1.yaml',
+6 -3
View File
@@ -20,7 +20,6 @@ mkMesonDerivation (finalAttrs: {
fileset = lib.fileset.unions [ fileset = lib.fileset.unions [
../../.version ../../.version
../../doc/manual/source/protocols/json/schema ../../doc/manual/source/protocols/json/schema
../../src/libutil-tests/data/memory-source-accessor
../../src/libutil-tests/data/hash ../../src/libutil-tests/data/hash
../../src/libstore-tests/data/content-address ../../src/libstore-tests/data/content-address
../../src/libstore-tests/data/store-path ../../src/libstore-tests/data/store-path
@@ -35,11 +34,15 @@ mkMesonDerivation (finalAttrs: {
outputs = [ "out" ]; outputs = [ "out" ];
passthru.externalNativeBuildInputs = [
jsonschema
];
nativeBuildInputs = [ nativeBuildInputs = [
meson meson
ninja ninja
jsonschema ]
]; ++ finalAttrs.passthru.externalNativeBuildInputs;
doCheck = true; doCheck = true;
+16 -10
View File
@@ -37,15 +37,7 @@ mkMesonDerivation (finalAttrs: {
outputs = [ "out" ]; outputs = [ "out" ];
buildInputs = [ passthru.externalNativeBuildInputs = [
gtest
kaitai-struct-cpp-stl-runtime
];
nativeBuildInputs = [
meson
ninja
pkg-config
# This can go away when we bump up to 25.11 # This can go away when we bump up to 25.11
(kaitai-struct-compiler.overrideAttrs (finalAttrs: { (kaitai-struct-compiler.overrideAttrs (finalAttrs: {
version = "0.11"; version = "0.11";
@@ -56,6 +48,20 @@ mkMesonDerivation (finalAttrs: {
})) }))
]; ];
passthru.externalBuildInputs = [
gtest
kaitai-struct-cpp-stl-runtime
];
buildInputs = finalAttrs.passthru.externalBuildInputs;
nativeBuildInputs = [
meson
ninja
pkg-config
]
++ finalAttrs.passthru.externalNativeBuildInputs;
doCheck = true; doCheck = true;
mesonCheckFlags = [ "--print-errorlogs" ]; mesonCheckFlags = [ "--print-errorlogs" ];
@@ -65,6 +71,6 @@ mkMesonDerivation (finalAttrs: {
''; '';
meta = { meta = {
platforms = lib.platforms.unix; platforms = lib.platforms.all;
}; };
}) })
-13
View File
@@ -2,7 +2,6 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include "nix/cmd/command.hh" #include "nix/cmd/command.hh"
#include "nix/cmd/legacy.hh"
#include "nix/cmd/markdown.hh" #include "nix/cmd/markdown.hh"
#include "nix/store/store-open.hh" #include "nix/store/store-open.hh"
#include "nix/store/local-fs-store.hh" #include "nix/store/local-fs-store.hh"
@@ -15,18 +14,6 @@
namespace nix { namespace nix {
RegisterCommand::Commands & RegisterCommand::commands()
{
static RegisterCommand::Commands commands;
return commands;
}
RegisterLegacyCommand::Commands & RegisterLegacyCommand::commands()
{
static RegisterLegacyCommand::Commands commands;
return commands;
}
nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix) nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
{ {
nix::Commands res; nix::Commands res;
+6 -8
View File
@@ -33,8 +33,7 @@ EvalSettings evalSettings{
// FIXME `parseFlakeRef` should take a `std::string_view`. // FIXME `parseFlakeRef` should take a `std::string_view`.
auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false);
debug("fetching flake search path element '%s''", rest); debug("fetching flake search path element '%s''", rest);
auto [accessor, lockedRef] = auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
auto storePath = nix::fetchToStore( auto storePath = nix::fetchToStore(
state.fetchSettings, state.fetchSettings,
*state.store, *state.store,
@@ -168,9 +167,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
? state.rootPath(absPath(getCommandBaseDir())) ? state.rootPath(absPath(getCommandBaseDir()))
: state.rootPath("."))); : state.rootPath(".")));
}, },
[&](const AutoArgString & arg) { v->mkString(arg.s, state.mem); }, [&](const AutoArgString & arg) { v->mkString(arg.s); },
[&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string()), state.mem); }, [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path.string())); },
[&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO), state.mem); }}, [&](const AutoArgStdin & arg) { v->mkString(readFile(STDIN_FILENO)); }},
arg); arg);
res.insert(state.symbols.create(name), v); res.insert(state.symbols.create(name), v);
} }
@@ -180,7 +179,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir) SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
{ {
if (EvalSettings::isPseudoUrl(s)) { if (EvalSettings::isPseudoUrl(s)) {
auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s)); auto accessor = fetchers::downloadTarball(state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy);
return state.storePath(storePath); return state.storePath(storePath);
} }
@@ -188,8 +187,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
else if (hasPrefix(s, "flake:")) { else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
auto [accessor, lockedRef] = auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store);
flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
auto storePath = nix::fetchToStore( auto storePath = nix::fetchToStore(
state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
state.allowPath(storePath); state.allowPath(storePath);
+5 -1
View File
@@ -286,7 +286,11 @@ struct RegisterCommand
{ {
typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands; typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
static Commands & commands(); static Commands & commands()
{
static Commands commands;
return commands;
}
RegisterCommand(std::vector<std::string> && name, std::function<ref<Command>()> command) RegisterCommand(std::vector<std::string> && name, std::function<ref<Command>()> command)
{ {
+5 -1
View File
@@ -13,7 +13,11 @@ struct RegisterLegacyCommand
{ {
typedef std::map<std::string, MainFunction> Commands; typedef std::map<std::string, MainFunction> Commands;
static Commands & commands(); static Commands & commands()
{
static Commands commands;
return commands;
}
RegisterLegacyCommand(const std::string & name, MainFunction fun) RegisterLegacyCommand(const std::string & name, MainFunction fun)
{ {
+1 -1
View File
@@ -409,7 +409,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
Args::completeDir(completions, 0, prefix); Args::completeDir(completions, 0, prefix);
/* Look for registry entries that match the prefix. */ /* Look for registry entries that match the prefix. */
for (auto & registry : fetchers::getRegistries(fetchSettings, *store)) { for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
for (auto & entry : registry->entries) { for (auto & entry : registry->entries) {
auto from = entry.from.to_string(); auto from = entry.from.to_string();
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
+2 -2
View File
@@ -738,8 +738,8 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
} }
auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true); auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings)) if (evalSettings.pureEval && !flakeRef.input.isLocked())
throw Error("cannot use ':load-flake' on unlocked flake reference '%s' (use --impure to override)", flakeRefS); throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
Value v; Value v;
+8 -18
View File
@@ -69,8 +69,8 @@ nix_err nix_expr_eval_from_string(
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
state->state.eval(parsedExpr, *value->value); state->state.eval(parsedExpr, value->value);
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -80,8 +80,8 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
state->state.callFunction(*fn->value, *arg->value, *value->value, nix::noPos); state->state.callFunction(fn->value, arg->value, value->value, nix::noPos);
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -91,15 +91,9 @@ nix_err nix_value_call_multi(
{ {
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
std::vector<nix::Value *> internal_args;
internal_args.reserve(nargs);
for (size_t i = 0; i < nargs; i++)
internal_args.push_back(args[i]->value);
try { try {
state->state.callFunction(*fn->value, {internal_args.data(), nargs}, *value->value, nix::noPos); state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos);
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -109,7 +103,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value *
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
state->state.forceValue(*value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -119,7 +113,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
state->state.forceValueDeep(*value->value); state->state.forceValueDeep(value->value);
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -143,8 +137,6 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto
void nix_eval_state_builder_free(nix_eval_state_builder * builder) void nix_eval_state_builder_free(nix_eval_state_builder * builder)
{ {
if (builder)
builder->~nix_eval_state_builder();
operator delete(builder, static_cast<std::align_val_t>(alignof(nix_eval_state_builder))); operator delete(builder, static_cast<std::align_val_t>(alignof(nix_eval_state_builder)));
} }
@@ -211,8 +203,6 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c
void nix_state_free(EvalState * state) void nix_state_free(EvalState * state)
{ {
if (state)
state->~EvalState();
operator delete(state, static_cast<std::align_val_t>(alignof(EvalState))); operator delete(state, static_cast<std::align_val_t>(alignof(EvalState)));
} }
+1 -7
View File
@@ -39,13 +39,7 @@ struct ListBuilder
struct nix_value struct nix_value
{ {
nix::Value * value; nix::Value value;
/**
* As we move to a managed heap, we need EvalMemory in more places. Ideally, we would take in EvalState or
* EvalMemory as an argument when we need it, but we don't want to make changes to the stable C api, so we stuff it
* into the nix_value that will get passed in to the relevant functions.
*/
nix::EvalMemory * mem;
}; };
struct nix_string_return struct nix_string_return
+25 -37
View File
@@ -20,7 +20,7 @@ static const nix::Value & check_value_not_null(const nix_value * value)
if (!value) { if (!value) {
throw std::runtime_error("nix_value is null"); throw std::runtime_error("nix_value is null");
} }
return *value->value; return *((const nix::Value *) value);
} }
static nix::Value & check_value_not_null(nix_value * value) static nix::Value & check_value_not_null(nix_value * value)
@@ -28,7 +28,7 @@ static nix::Value & check_value_not_null(nix_value * value)
if (!value) { if (!value) {
throw std::runtime_error("nix_value is null"); throw std::runtime_error("nix_value is null");
} }
return *value->value; return value->value;
} }
static const nix::Value & check_value_in(const nix_value * value) static const nix::Value & check_value_in(const nix_value * value)
@@ -58,14 +58,9 @@ static nix::Value & check_value_out(nix_value * value)
return v; return v;
} }
static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem) static inline nix_value * as_nix_value_ptr(nix::Value * v)
{ {
nix_value * ret = new (mem.allocBytes(sizeof(nix_value))) nix_value{ return reinterpret_cast<nix_value *>(v);
.value = v,
.mem = &mem,
};
nix_gc_incref(nullptr, ret);
return ret;
} }
/** /**
@@ -74,13 +69,7 @@ static inline nix_value * new_nix_value(nix::Value * v, nix::EvalMemory & mem)
* Deals with errors and converts arguments from C++ into C types. * Deals with errors and converts arguments from C++ into C types.
*/ */
static void nix_c_primop_wrapper( static void nix_c_primop_wrapper(
PrimOpFun f, PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
void * userdata,
int arity,
nix::EvalState & state,
const nix::PosIdx pos,
nix::Value ** args,
nix::Value & v)
{ {
nix_c_context ctx; nix_c_context ctx;
@@ -96,15 +85,8 @@ static void nix_c_primop_wrapper(
// ok because we don't see a need for this yet (e.g. inspecting thunks, // ok because we don't see a need for this yet (e.g. inspecting thunks,
// or maybe something to make blackholes work better; we don't know). // or maybe something to make blackholes work better; we don't know).
nix::Value vTmp; nix::Value vTmp;
nix_value * vTmpPtr = new_nix_value(&vTmp, state.mem);
std::vector<nix_value *> external_args; f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp);
external_args.reserve(arity);
for (int i = 0; i < arity; i++) {
nix_value * external_arg = new_nix_value(args[i], state.mem);
external_args.push_back(external_arg);
}
f(userdata, &ctx, (EvalState *) &state, external_args.data(), vTmpPtr);
if (ctx.last_err_code != NIX_OK) { if (ctx.last_err_code != NIX_OK) {
/* TODO: Throw different errors depending on the error code */ /* TODO: Throw different errors depending on the error code */
@@ -153,7 +135,7 @@ PrimOp * nix_alloc_primop(
.args = {}, .args = {},
.arity = (size_t) arity, .arity = (size_t) arity,
.doc = doc, .doc = doc,
.fun = std::bind(nix_c_primop_wrapper, fun, user_data, arity, _1, _2, _3, _4)}; .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
if (args) if (args)
for (size_t i = 0; args[i]; i++) for (size_t i = 0; args[i]; i++)
p->args.emplace_back(*args); p->args.emplace_back(*args);
@@ -178,7 +160,8 @@ nix_value * nix_alloc_value(nix_c_context * context, EvalState * state)
if (context) if (context)
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
nix_value * res = new_nix_value(state->state.allocValue(), state->state.mem); nix_value * res = as_nix_value_ptr(state->state.allocValue());
nix_gc_incref(nullptr, res);
return res; return res;
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
@@ -348,10 +331,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value,
return nullptr; return nullptr;
} }
auto * p = v.listView()[ix]; auto * p = v.listView()[ix];
if (p == nullptr) nix_gc_incref(nullptr, p);
return nullptr; if (p != nullptr)
state->state.forceValue(*p, nix::noPos); state->state.forceValue(*p, nix::noPos);
return new_nix_value(p, state->state.mem); return as_nix_value_ptr(p);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -369,8 +352,9 @@ nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalSt
return nullptr; return nullptr;
} }
auto * p = v.listView()[ix]; auto * p = v.listView()[ix];
nix_gc_incref(nullptr, p);
// Note: intentionally NOT calling forceValue() to keep the element lazy // Note: intentionally NOT calling forceValue() to keep the element lazy
return new_nix_value(p, state->state.mem); return as_nix_value_ptr(p);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -385,8 +369,9 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value
nix::Symbol s = state->state.symbols.create(name); nix::Symbol s = state->state.symbols.create(name);
auto attr = v.attrs()->get(s); auto attr = v.attrs()->get(s);
if (attr) { if (attr) {
nix_gc_incref(nullptr, attr->value);
state->state.forceValue(*attr->value, nix::noPos); state->state.forceValue(*attr->value, nix::noPos);
return new_nix_value(attr->value, state->state.mem); return as_nix_value_ptr(attr->value);
} }
nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
return nullptr; return nullptr;
@@ -405,8 +390,9 @@ nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalS
nix::Symbol s = state->state.symbols.create(name); nix::Symbol s = state->state.symbols.create(name);
auto attr = v.attrs()->get(s); auto attr = v.attrs()->get(s);
if (attr) { if (attr) {
nix_gc_incref(nullptr, attr->value);
// Note: intentionally NOT calling forceValue() to keep the attribute lazy // Note: intentionally NOT calling forceValue() to keep the attribute lazy
return new_nix_value(attr->value, state->state.mem); return as_nix_value_ptr(attr->value);
} }
nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
return nullptr; return nullptr;
@@ -454,8 +440,9 @@ nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state
} }
const nix::Attr & a = (*v.attrs())[i]; const nix::Attr & a = (*v.attrs())[i];
*name = state->state.symbols[a.name].c_str(); *name = state->state.symbols[a.name].c_str();
nix_gc_incref(nullptr, a.value);
state->state.forceValue(*a.value, nix::noPos); state->state.forceValue(*a.value, nix::noPos);
return new_nix_value(a.value, state->state.mem); return as_nix_value_ptr(a.value);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -474,8 +461,9 @@ nix_value * nix_get_attr_byidx_lazy(
} }
const nix::Attr & a = (*v.attrs())[i]; const nix::Attr & a = (*v.attrs())[i];
*name = state->state.symbols[a.name].c_str(); *name = state->state.symbols[a.name].c_str();
nix_gc_incref(nullptr, a.value);
// Note: intentionally NOT calling forceValue() to keep the attribute lazy // Note: intentionally NOT calling forceValue() to keep the attribute lazy
return new_nix_value(a.value, state->state.mem); return as_nix_value_ptr(a.value);
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
} }
@@ -515,7 +503,7 @@ nix_err nix_init_string(nix_c_context * context, nix_value * value, const char *
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
auto & v = check_value_out(value); auto & v = check_value_out(value);
v.mkString(std::string_view(str), *value->mem); v.mkString(std::string_view(str));
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -526,7 +514,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value *
context->last_err_code = NIX_OK; context->last_err_code = NIX_OK;
try { try {
auto & v = check_value_out(value); auto & v = check_value_out(value);
v.mkPath(s->state.rootPath(nix::CanonPath(str)), s->state.mem); v.mkPath(s->state.rootPath(nix::CanonPath(str)));
} }
NIXC_CATCH_ERRS NIXC_CATCH_ERRS
} }
@@ -104,7 +104,6 @@ MATCHER(IsAttrs, "")
MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s)) MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s))
{ {
if (arg.type() != nString) { if (arg.type() != nString) {
*result_listener << "Expected a string got " << arg.type();
return false; return false;
} }
return arg.string_view() == s; return arg.string_view() == s;
+3 -4
View File
@@ -1,6 +1,5 @@
#include "nix/expr/tests/libexpr.hh" #include "nix/expr/tests/libexpr.hh"
#include "nix/expr/value-to-json.hh" #include "nix/expr/value-to-json.hh"
#include "nix/expr/static-string-data.hh"
namespace nix { namespace nix {
// Testing the conversion to JSON // Testing the conversion to JSON
@@ -55,7 +54,7 @@ TEST_F(JSONValueTest, IntNegative)
TEST_F(JSONValueTest, String) TEST_F(JSONValueTest, String)
{ {
Value v; Value v;
v.mkStringNoCopy("test"_sds); v.mkStringNoCopy("test");
ASSERT_EQ(getJSONValue(v), "\"test\""); ASSERT_EQ(getJSONValue(v), "\"test\"");
} }
@@ -63,7 +62,7 @@ TEST_F(JSONValueTest, StringQuotes)
{ {
Value v; Value v;
v.mkStringNoCopy("test\""_sds); v.mkStringNoCopy("test\"");
ASSERT_EQ(getJSONValue(v), "\"test\\\"\""); ASSERT_EQ(getJSONValue(v), "\"test\\\"\"");
} }
@@ -73,7 +72,7 @@ TEST_F(JSONValueTest, StringQuotes)
TEST_F(JSONValueTest, DISABLED_Path) TEST_F(JSONValueTest, DISABLED_Path)
{ {
Value v; Value v;
v.mkPath(state.rootPath(CanonPath("/test")), state.mem); v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
} }
} /* namespace nix */ } /* namespace nix */
+1 -5
View File
@@ -14,7 +14,7 @@
namespace nixC { namespace nixC {
TEST_F(nix_api_expr_test, nix_eval_state_lookup_path) TEST_F(nix_api_store_test, nix_eval_state_lookup_path)
{ {
auto tmpDir = nix::createTempDir(); auto tmpDir = nix::createTempDir();
auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true); auto delTmpDir = std::make_unique<nix::AutoDelete>(tmpDir, true);
@@ -42,16 +42,12 @@ TEST_F(nix_api_expr_test, nix_eval_state_lookup_path)
nix_expr_eval_from_string(ctx, state, "builtins.seq <nixos-config> <nixpkgs>", ".", value); nix_expr_eval_from_string(ctx, state, "builtins.seq <nixos-config> <nixpkgs>", ".", value);
assert_ctx_ok(); assert_ctx_ok();
nix_state_free(state);
ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH); ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH);
assert_ctx_ok(); assert_ctx_ok();
auto pathStr = nix_get_path_string(ctx, value); auto pathStr = nix_get_path_string(ctx, value);
assert_ctx_ok(); assert_ctx_ok();
ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str())); ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str()));
nix_gc_decref(nullptr, value);
} }
TEST_F(nix_api_expr_test, nix_expr_eval_from_string) TEST_F(nix_api_expr_test, nix_expr_eval_from_string)
+2 -8
View File
@@ -661,14 +661,8 @@ INSTANTIATE_TEST_SUITE_P(
CASE(R"(null)", ""), CASE(R"(null)", ""),
CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"), CASE(R"({ v = "bar"; __toString = self: self.v; })", "bar"),
CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"), CASE(R"({ v = "bar"; __toString = self: self.v; outPath = "foo"; })", "bar"),
CASE(R"({ outPath = "foo"; })", "foo") CASE(R"({ outPath = "foo"; })", "foo"),
// this is broken on cygwin because canonPath("//./test", false) returns //./test CASE(R"(./test)", "/test")));
// FIXME: don't use canonPath
#ifndef __CYGWIN__
,
CASE(R"(./test)", "/test")
#endif
));
#undef CASE #undef CASE
TEST_F(PrimOpTest, substring) TEST_F(PrimOpTest, substring)
+12 -13
View File
@@ -1,5 +1,4 @@
#include "nix/expr/tests/libexpr.hh" #include "nix/expr/tests/libexpr.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/expr/value.hh" #include "nix/expr/value.hh"
#include "nix/expr/print.hh" #include "nix/expr/print.hh"
@@ -36,14 +35,14 @@ TEST_F(ValuePrintingTests, tBool)
TEST_F(ValuePrintingTests, tString) TEST_F(ValuePrintingTests, tString)
{ {
Value vString; Value vString;
vString.mkStringNoCopy("some-string"_sds); vString.mkStringNoCopy("some-string");
test(vString, "\"some-string\""); test(vString, "\"some-string\"");
} }
TEST_F(ValuePrintingTests, tPath) TEST_F(ValuePrintingTests, tPath)
{ {
Value vPath; Value vPath;
vPath.mkStringNoCopy("/foo"_sds); vPath.mkStringNoCopy("/foo");
test(vPath, "\"/foo\""); test(vPath, "\"/foo\"");
} }
@@ -268,7 +267,7 @@ struct StringPrintingTests : LibExprTest
void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args) void test(std::string_view literal, std::string_view expected, unsigned int maxLength, A... args)
{ {
Value v; Value v;
v.mkString(literal, state.mem); v.mkString(literal);
std::stringstream out; std::stringstream out;
printValue(state, out, v, PrintOptions{.maxStringLength = maxLength}); printValue(state, out, v, PrintOptions{.maxStringLength = maxLength});
@@ -290,10 +289,10 @@ TEST_F(StringPrintingTests, maxLengthTruncation)
TEST_F(ValuePrintingTests, attrsTypeFirst) TEST_F(ValuePrintingTests, attrsTypeFirst)
{ {
Value vType; Value vType;
vType.mkStringNoCopy("puppy"_sds); vType.mkStringNoCopy("puppy");
Value vApple; Value vApple;
vApple.mkStringNoCopy("apple"_sds); vApple.mkStringNoCopy("apple");
BindingsBuilder builder = state.buildBindings(10); BindingsBuilder builder = state.buildBindings(10);
builder.insert(state.symbols.create("type"), &vType); builder.insert(state.symbols.create("type"), &vType);
@@ -334,7 +333,7 @@ TEST_F(ValuePrintingTests, ansiColorsBool)
TEST_F(ValuePrintingTests, ansiColorsString) TEST_F(ValuePrintingTests, ansiColorsString)
{ {
Value v; Value v;
v.mkStringNoCopy("puppy"_sds); v.mkStringNoCopy("puppy");
test(v, ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, PrintOptions{.ansiColors = true}); test(v, ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, PrintOptions{.ansiColors = true});
} }
@@ -342,7 +341,7 @@ TEST_F(ValuePrintingTests, ansiColorsString)
TEST_F(ValuePrintingTests, ansiColorsStringElided) TEST_F(ValuePrintingTests, ansiColorsStringElided)
{ {
Value v; Value v;
v.mkStringNoCopy("puppy"_sds); v.mkStringNoCopy("puppy");
test( test(
v, v,
@@ -353,7 +352,7 @@ TEST_F(ValuePrintingTests, ansiColorsStringElided)
TEST_F(ValuePrintingTests, ansiColorsPath) TEST_F(ValuePrintingTests, ansiColorsPath)
{ {
Value v; Value v;
v.mkPath(state.rootPath(CanonPath("puppy")), state.mem); v.mkPath(state.rootPath(CanonPath("puppy")));
test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true}); test(v, ANSI_GREEN "/puppy" ANSI_NORMAL, PrintOptions{.ansiColors = true});
} }
@@ -390,7 +389,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs)
TEST_F(ValuePrintingTests, ansiColorsDerivation) TEST_F(ValuePrintingTests, ansiColorsDerivation)
{ {
Value vDerivation; Value vDerivation;
vDerivation.mkStringNoCopy("derivation"_sds); vDerivation.mkStringNoCopy("derivation");
BindingsBuilder builder = state.buildBindings(10); BindingsBuilder builder = state.buildBindings(10);
builder.insert(state.s.type, &vDerivation); builder.insert(state.s.type, &vDerivation);
@@ -413,7 +412,7 @@ TEST_F(ValuePrintingTests, ansiColorsError)
{ {
Value throw_ = state.getBuiltin("throw"); Value throw_ = state.getBuiltin("throw");
Value message; Value message;
message.mkStringNoCopy("uh oh!"_sds); message.mkStringNoCopy("uh oh!");
Value vError; Value vError;
vError.mkApp(&throw_, &message); vError.mkApp(&throw_, &message);
@@ -430,12 +429,12 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError)
{ {
Value throw_ = state.getBuiltin("throw"); Value throw_ = state.getBuiltin("throw");
Value message; Value message;
message.mkStringNoCopy("uh oh!"_sds); message.mkStringNoCopy("uh oh!");
Value vError; Value vError;
vError.mkApp(&throw_, &message); vError.mkApp(&throw_, &message);
Value vDerivation; Value vDerivation;
vDerivation.mkStringNoCopy("derivation"_sds); vDerivation.mkStringNoCopy("derivation");
BindingsBuilder builder = state.buildBindings(10); BindingsBuilder builder = state.buildBindings(10);
builder.insert(state.s.type, &vDerivation); builder.insert(state.s.type, &vDerivation);
+6 -7
View File
@@ -1,5 +1,4 @@
#include "nix/expr/value.hh" #include "nix/expr/value.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/store/tests/libstore.hh" #include "nix/store/tests/libstore.hh"
#include <gtest/gtest.h> #include <gtest/gtest.h>
@@ -28,17 +27,17 @@ TEST_F(ValueTest, staticString)
{ {
Value vStr1; Value vStr1;
Value vStr2; Value vStr2;
vStr1.mkStringNoCopy("foo"_sds); vStr1.mkStringNoCopy("foo");
vStr2.mkStringNoCopy("foo"_sds); vStr2.mkStringNoCopy("foo");
auto & sd1 = vStr1.string_data(); auto sd1 = vStr1.string_view();
auto & sd2 = vStr2.string_data(); auto sd2 = vStr2.string_view();
// The strings should be the same // The strings should be the same
ASSERT_EQ(sd1.view(), sd2.view()); ASSERT_EQ(sd1, sd2);
// The strings should also be backed by the same (static) allocation // The strings should also be backed by the same (static) allocation
ASSERT_EQ(&sd1, &sd2); ASSERT_EQ(sd1.data(), sd2.data());
} }
} // namespace nix } // namespace nix
+1 -1
View File
@@ -147,7 +147,7 @@ struct AttrDb
for (auto * elem : *context) { for (auto * elem : *context) {
if (!first) if (!first)
ctx.push_back(' '); ctx.push_back(' ');
ctx.append(elem->view()); ctx.append(elem);
first = false; first = false;
} }
state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx) state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx)
+54 -98
View File
@@ -3,7 +3,6 @@
#include "nix/expr/primops.hh" #include "nix/expr/primops.hh"
#include "nix/expr/print-options.hh" #include "nix/expr/print-options.hh"
#include "nix/expr/symbol-table.hh" #include "nix/expr/symbol-table.hh"
#include "nix/expr/value.hh"
#include "nix/util/exit.hh" #include "nix/util/exit.hh"
#include "nix/util/types.hh" #include "nix/util/types.hh"
#include "nix/util/util.hh" #include "nix/util/util.hh"
@@ -29,8 +28,6 @@
#include "parser-tab.hh" #include "parser-tab.hh"
#include <algorithm> #include <algorithm>
#include <cstddef>
#include <cstdlib>
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
#include <cstring> #include <cstring>
@@ -51,9 +48,6 @@ using json = nlohmann::json;
namespace nix { namespace nix {
/**
* Just for doc strings. Not for regular string values.
*/
static char * allocString(size_t size) static char * allocString(size_t size)
{ {
char * t; char * t;
@@ -67,9 +61,6 @@ static char * allocString(size_t size)
// string allocations. // string allocations.
// This function handles makeImmutableString(std::string_view()) by returning // This function handles makeImmutableString(std::string_view()) by returning
// the empty string. // the empty string.
/**
* Just for doc strings. Not for regular string values.
*/
static const char * makeImmutableString(std::string_view s) static const char * makeImmutableString(std::string_view s)
{ {
const size_t size = s.size(); const size_t size = s.size();
@@ -81,25 +72,6 @@ static const char * makeImmutableString(std::string_view s)
return t; return t;
} }
StringData & StringData::alloc(EvalMemory & mem, size_t size)
{
void * t = mem.allocBytes(sizeof(StringData) + size + 1);
if (!t)
throw std::bad_alloc();
auto res = new (t) StringData(size);
return *res;
}
const StringData & StringData::make(EvalMemory & mem, std::string_view s)
{
if (s.empty())
return ""_sds;
auto & res = alloc(mem, s.size());
std::memcpy(&res.data_, s.data(), s.size());
res.data_[s.size()] = '\0';
return res;
}
RootValue allocRootValue(Value * v) RootValue allocRootValue(Value * v)
{ {
return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v); return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
@@ -205,7 +177,7 @@ bool Value::isTrivial() const
{ {
return !isa<tApp, tPrimOpApp>() return !isa<tApp, tPrimOpApp>()
&& (!isa<tThunk>() && (!isa<tThunk>()
|| (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs->empty()) || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty())
|| dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr)); || dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr));
} }
@@ -517,16 +489,15 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
if (primOp.arity == 0) { if (primOp.arity == 0) {
primOp.arity = 1; primOp.arity = 1;
auto vPrimOp = allocValue(); auto vPrimOp = allocValue();
vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); vPrimOp->mkPrimOp(new PrimOp(primOp));
Value v; Value v;
v.mkApp(vPrimOp, vPrimOp); v.mkApp(vPrimOp, vPrimOp);
auto & primOp1 = *vPrimOp->primOp();
return addConstant( return addConstant(
primOp1.name, primOp.name,
v, v,
{ {
.type = nThunk, // FIXME .type = nThunk, // FIXME
.doc = primOp1.doc ? primOp1.doc->c_str() : nullptr, .doc = primOp.doc,
}); });
} }
@@ -566,14 +537,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{ {
if (v.isPrimOp()) { if (v.isPrimOp()) {
auto v2 = &v; auto v2 = &v;
auto & primOp = *v2->primOp(); if (auto * doc = v2->primOp()->doc)
if (primOp.doc)
return Doc{ return Doc{
.pos = {}, .pos = {},
.name = primOp.name, .name = v2->primOp()->name,
.arity = primOp.arity, .arity = v2->primOp()->arity,
.args = primOp.args, .args = v2->primOp()->args,
.doc = primOp.doc->c_str(), .doc = doc,
}; };
} }
if (v.isLambda()) { if (v.isLambda()) {
@@ -615,9 +585,7 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
.name = name, .name = name,
.arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
.args = {}, .args = {},
/* N.B. Can't use StringData here, because that would lead to an interior pointer. .doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC
NOTE: memory leak when compiled without GC. */
.doc = makeImmutableString(s.view()),
}; };
} }
if (isFunctor(v)) { if (isFunctor(v)) {
@@ -849,36 +817,35 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
evalState.runDebugRepl(nullptr, trace.env, trace.expr); evalState.runDebugRepl(nullptr, trace.env, trace.expr);
} }
void Value::mkString(std::string_view s, EvalMemory & mem) void Value::mkString(std::string_view s)
{ {
mkStringNoCopy(StringData::make(mem, s)); mkStringNoCopy(makeImmutableString(s));
} }
Value::StringWithContext::Context * Value::StringWithContext::Context * Value::StringWithContext::Context::fromBuilder(const NixStringContext & context)
Value::StringWithContext::Context::fromBuilder(const NixStringContext & context, EvalMemory & mem)
{ {
if (context.empty()) if (context.empty())
return nullptr; return nullptr;
auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size()); auto ctx = new (allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
std::ranges::transform( std::ranges::transform(
context, ctx->elems, [&](const NixStringContextElem & elt) { return &StringData::make(mem, elt.to_string()); }); context, ctx->elems, [](const NixStringContextElem & elt) { return makeImmutableString(elt.to_string()); });
return ctx; return ctx;
} }
void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem) void Value::mkString(std::string_view s, const NixStringContext & context)
{ {
mkStringNoCopy(StringData::make(mem, s), Value::StringWithContext::Context::fromBuilder(context, mem)); mkStringNoCopy(makeImmutableString(s), Value::StringWithContext::Context::fromBuilder(context));
} }
void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem) void Value::mkStringMove(const char * s, const NixStringContext & context)
{ {
mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem)); mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context));
} }
void Value::mkPath(const SourcePath & path, EvalMemory & mem) void Value::mkPath(const SourcePath & path)
{ {
mkPath(&*path.accessor, StringData::make(mem, path.path.abs())); mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
} }
inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -914,9 +881,9 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
} }
} }
ListBuilder::ListBuilder(EvalMemory & mem, size_t size) ListBuilder::ListBuilder(size_t size)
: size(size) : size(size)
, elems(size <= 2 ? inlineElems : (Value **) mem.allocBytes(size * sizeof(Value *))) , elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *)))
{ {
} }
@@ -943,7 +910,7 @@ void EvalState::mkPos(Value & v, PosIdx p)
auto origin = positions.originOf(p); auto origin = positions.originOf(p);
if (auto path = std::get_if<SourcePath>(&origin)) { if (auto path = std::get_if<SourcePath>(&origin)) {
auto attrs = buildBindings(3); auto attrs = buildBindings(3);
attrs.alloc(s.file).mkString(path->path.abs(), mem); attrs.alloc(s.file).mkString(path->path.abs());
makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column)); makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column));
v.mkAttrs(attrs); v.mkAttrs(attrs);
} else } else
@@ -956,8 +923,7 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
store->printStorePath(p), store->printStorePath(p),
NixStringContext{ NixStringContext{
NixStringContextElem::Opaque{.path = p}, NixStringContextElem::Opaque{.path = p},
}, });
mem);
} }
std::string EvalState::mkOutputStringRaw( std::string EvalState::mkOutputStringRaw(
@@ -979,7 +945,7 @@ void EvalState::mkOutputString(
std::optional<StorePath> optStaticOutputPath, std::optional<StorePath> optStaticOutputPath,
const ExperimentalFeatureSettings & xpSettings) const ExperimentalFeatureSettings & xpSettings)
{ {
value.mkString(mkOutputStringRaw(b, optStaticOutputPath, xpSettings), NixStringContext{b}, mem); value.mkString(mkOutputStringRaw(b, optStaticOutputPath, xpSettings), NixStringContext{b});
} }
std::string EvalState::mkSingleDerivedPathStringRaw(const SingleDerivedPath & p) std::string EvalState::mkSingleDerivedPathStringRaw(const SingleDerivedPath & p)
@@ -1014,8 +980,7 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v
mkSingleDerivedPathStringRaw(p), mkSingleDerivedPathStringRaw(p),
NixStringContext{ NixStringContext{
std::visit([](auto && v) -> NixStringContextElem { return v; }, p), std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
}, });
mem);
} }
Value * Expr::maybeThunk(EvalState & state, Env & env) Value * Expr::maybeThunk(EvalState & state, Env & env)
@@ -1226,26 +1191,26 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
void ExprAttrs::eval(EvalState & state, Env & env, Value & v) void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
{ {
auto bindings = state.buildBindings(attrs->size() + dynamicAttrs->size()); auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
auto dynamicEnv = &env; auto dynamicEnv = &env;
bool sort = false; bool sort = false;
if (recursive) { if (recursive) {
/* Create a new environment that contains the attributes in /* Create a new environment that contains the attributes in
this `rec'. */ this `rec'. */
Env & env2(state.mem.allocEnv(attrs->size())); Env & env2(state.mem.allocEnv(attrs.size()));
env2.up = &env; env2.up = &env;
dynamicEnv = &env2; dynamicEnv = &env2;
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
AttrDefs::iterator overrides = attrs->find(state.s.overrides); AttrDefs::iterator overrides = attrs.find(state.s.overrides);
bool hasOverrides = overrides != attrs->end(); bool hasOverrides = overrides != attrs.end();
/* The recursive attributes are evaluated in the new /* The recursive attributes are evaluated in the new
environment, while the inherited attributes are evaluated environment, while the inherited attributes are evaluated
in the original environment. */ in the original environment. */
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs) { for (auto & i : attrs) {
Value * vAttr; Value * vAttr;
if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) { if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
vAttr = state.allocValue(); vAttr = state.allocValue();
@@ -1272,8 +1237,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
"while evaluating the `__overrides` attribute"); "while evaluating the `__overrides` attribute");
bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size())); bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size()));
for (auto & i : *vOverrides->attrs()) { for (auto & i : *vOverrides->attrs()) {
AttrDefs::iterator j = attrs->find(i.name); AttrDefs::iterator j = attrs.find(i.name);
if (j != attrs->end()) { if (j != attrs.end()) {
(*bindings.bindings)[j->second.displ] = i; (*bindings.bindings)[j->second.displ] = i;
env2.values[j->second.displ] = i.value; env2.values[j->second.displ] = i.value;
} else } else
@@ -1285,13 +1250,13 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
else { else {
Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
for (auto & i : *attrs) for (auto & i : attrs)
bindings.insert( bindings.insert(
i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos); i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos);
} }
/* Dynamic attrs apply *after* rec and __overrides. */ /* Dynamic attrs apply *after* rec and __overrides. */
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
Value nameVal; Value nameVal;
i.nameExpr->eval(state, *dynamicEnv, nameVal); i.nameExpr->eval(state, *dynamicEnv, nameVal);
state.forceValue(nameVal, i.pos); state.forceValue(nameVal, i.pos);
@@ -1325,7 +1290,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
{ {
/* Create a new environment that contains the attributes in this /* Create a new environment that contains the attributes in this
`let'. */ `let'. */
Env & env2(state.mem.allocEnv(attrs->attrs->size())); Env & env2(state.mem.allocEnv(attrs->attrs.size()));
env2.up = &env; env2.up = &env;
Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr; Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1334,7 +1299,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
while the inherited attributes are evaluated in the original while the inherited attributes are evaluated in the original
environment. */ environment. */
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs->attrs) { for (auto & i : attrs->attrs) {
env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
} }
@@ -1751,9 +1716,9 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
// 4: about 60 // 4: about 60
// 5: under 10 // 5: under 10
// This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total.
SmallValueVector<4> vArgs(args->size()); SmallValueVector<4> vArgs(args.size());
for (size_t i = 0; i < args->size(); ++i) for (size_t i = 0; i < args.size(); ++i)
vArgs[i] = (*args)[i]->maybeThunk(state, env); vArgs[i] = args[i]->maybeThunk(state, env);
state.callFunction(vFun, vArgs, v, pos); state.callFunction(vFun, vArgs, v, pos);
} }
@@ -2134,21 +2099,21 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
.atPos(pos) .atPos(pos)
.withFrame(env, *this) .withFrame(env, *this)
.debugThrow(); .debugThrow();
std::string resultStr; std::string result_str;
resultStr.reserve(sSize); result_str.reserve(sSize);
for (const auto & part : strings) { for (const auto & part : strings) {
resultStr += *part; result_str += *part;
} }
v.mkPath(state.rootPath(CanonPath(resultStr)), state.mem); v.mkPath(state.rootPath(CanonPath(result_str)));
} else { } else {
auto & resultStr = StringData::alloc(state.mem, sSize); char * result_str = allocString(sSize + 1);
auto * tmp = resultStr.data(); char * tmp = result_str;
for (const auto & part : strings) { for (const auto & part : strings) {
std::memcpy(tmp, part->data(), part->size()); memcpy(tmp, part->data(), part->size());
tmp += part->size(); tmp += part->size();
} }
*tmp = '\0'; *tmp = 0;
v.mkStringMove(resultStr, context, state.mem); v.mkStringMove(result_str, context);
} }
} }
@@ -2188,8 +2153,6 @@ void EvalState::forceValueDeep(Value & v)
std::set<const Value *> seen; std::set<const Value *> seen;
[&, &state(*this)](this const auto & recurse, Value & v) { [&, &state(*this)](this const auto & recurse, Value & v) {
auto _level = state.addCallDepth(v.determinePos(noPos));
if (!seen.insert(&v).second) if (!seen.insert(&v).second)
return; return;
@@ -2216,15 +2179,8 @@ void EvalState::forceValueDeep(Value & v)
} }
else if (v.isList()) { else if (v.isList()) {
size_t index = 0;
for (auto v2 : v.listView()) for (auto v2 : v.listView())
try { recurse(*v2);
recurse(*v2);
index++;
} catch (Error & e) {
state.addErrorTrace(e, "while evaluating list element at index %1%", index);
throw;
}
} }
}(v); }(v);
} }
@@ -2332,7 +2288,7 @@ void copyContext(const Value & v, NixStringContext & context, const Experimental
{ {
if (auto * ctx = v.context()) if (auto * ctx = v.context())
for (auto * elem : *ctx) for (auto * elem : *ctx)
context.insert(NixStringContextElem::parse(elem->view(), xpSettings)); context.insert(NixStringContextElem::parse(elem, xpSettings));
} }
std::string_view EvalState::forceString( std::string_view EvalState::forceString(
@@ -2354,7 +2310,7 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
error<EvalError>( error<EvalError>(
"the string '%1%' is not allowed to refer to a store path (such as '%2%')", "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
v.string_view(), v.string_view(),
(*v.context()->begin())->view()) *v.context()->begin())
.withTrace(pos, errorCtx) .withTrace(pos, errorCtx)
.debugThrow(); .debugThrow();
} }
@@ -3199,7 +3155,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
if (EvalSettings::isPseudoUrl(value)) { if (EvalSettings::isPseudoUrl(value)) {
try { try {
auto accessor = fetchers::downloadTarball(*store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); auto accessor = fetchers::downloadTarball(store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
return finish(this->storePath(storePath)); return finish(this->storePath(storePath));
} catch (Error & e) { } catch (Error & e) {
+1 -1
View File
@@ -12,7 +12,7 @@ namespace nix {
* Note: Various places expect the allocated memory to be zeroed. * Note: Various places expect the allocated memory to be zeroed.
*/ */
[[gnu::always_inline]] [[gnu::always_inline]]
inline void * EvalMemory::allocBytes(size_t n) inline void * allocBytes(size_t n)
{ {
void * p; void * p;
#if NIX_USE_BOEHMGC #if NIX_USE_BOEHMGC
+2 -3
View File
@@ -108,7 +108,7 @@ struct PrimOp
/** /**
* Optional free-form documentation about the primop. * Optional free-form documentation about the primop.
*/ */
std::optional<std::string> doc; const char * doc = nullptr;
/** /**
* Add a trace item, while calling the `<name>` builtin. * Add a trace item, while calling the `<name>` builtin.
@@ -335,7 +335,6 @@ public:
EvalMemory & operator=(const EvalMemory &) = delete; EvalMemory & operator=(const EvalMemory &) = delete;
EvalMemory & operator=(EvalMemory &&) = delete; EvalMemory & operator=(EvalMemory &&) = delete;
inline void * allocBytes(size_t n);
inline Value * allocValue(); inline Value * allocValue();
inline Env & allocEnv(size_t size); inline Env & allocEnv(size_t size);
@@ -349,7 +348,7 @@ public:
ListBuilder buildList(size_t size) ListBuilder buildList(size_t size)
{ {
stats.nrListElems += size; stats.nrListElems += size;
return ListBuilder(*this, size); return ListBuilder(size);
} }
const Statistics & getStats() const & const Statistics & getStats() const &
-1
View File
@@ -31,7 +31,6 @@ headers = [ config_pub_h ] + files(
'print.hh', 'print.hh',
'repl-exit-status.hh', 'repl-exit-status.hh',
'search-path.hh', 'search-path.hh',
'static-string-data.hh',
'symbol-table.hh', 'symbol-table.hh',
'value-to-json.hh', 'value-to-json.hh',
'value-to-xml.hh', 'value-to-xml.hh',
+30 -62
View File
@@ -3,7 +3,6 @@
#include <map> #include <map>
#include <span> #include <span>
#include <memory>
#include <vector> #include <vector>
#include <memory_resource> #include <memory_resource>
#include <algorithm> #include <algorithm>
@@ -12,7 +11,6 @@
#include "nix/expr/value.hh" #include "nix/expr/value.hh"
#include "nix/expr/symbol-table.hh" #include "nix/expr/symbol-table.hh"
#include "nix/expr/eval-error.hh" #include "nix/expr/eval-error.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/util/pos-idx.hh" #include "nix/util/pos-idx.hh"
#include "nix/expr/counter.hh" #include "nix/expr/counter.hh"
#include "nix/util/pos-table.hh" #include "nix/util/pos-table.hh"
@@ -188,18 +186,22 @@ struct ExprString : Expr
* This is only for strings already allocated in our polymorphic allocator, * This is only for strings already allocated in our polymorphic allocator,
* or that live at least that long (e.g. c++ string literals) * or that live at least that long (e.g. c++ string literals)
*/ */
ExprString(const StringData & s) ExprString(const char * s)
{ {
v.mkStringNoCopy(s); v.mkStringNoCopy(s);
}; };
ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv) ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv)
{ {
if (sv.size() == 0) { auto len = sv.length();
v.mkStringNoCopy(""_sds); if (len == 0) {
v.mkStringNoCopy("");
return; return;
} }
v.mkStringNoCopy(StringData::make(*alloc.resource(), sv)); char * s = alloc.allocate(len + 1);
sv.copy(s, len);
s[len] = '\0';
v.mkStringNoCopy(s);
}; };
Value * maybeThunk(EvalState & state, Env & env) override; Value * maybeThunk(EvalState & state, Env & env) override;
@@ -214,7 +216,11 @@ struct ExprPath : Expr
ExprPath(std::pmr::polymorphic_allocator<char> & alloc, ref<SourceAccessor> accessor, std::string_view sv) ExprPath(std::pmr::polymorphic_allocator<char> & alloc, ref<SourceAccessor> accessor, std::string_view sv)
: accessor(accessor) : accessor(accessor)
{ {
v.mkPath(&*accessor, StringData::make(*alloc.resource(), sv)); auto len = sv.length();
char * s = alloc.allocate(len + 1);
sv.copy(s, len);
s[len] = '\0';
v.mkPath(&*accessor, s);
} }
Value * maybeThunk(EvalState & state, Env & env) override; Value * maybeThunk(EvalState & state, Env & env) override;
@@ -339,7 +345,7 @@ struct ExprOpHasAttr : Expr
Expr * e; Expr * e;
std::span<AttrName> attrPath; std::span<AttrName> attrPath;
ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::span<AttrName> attrPath) ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::vector<AttrName> attrPath)
: e(e) : e(e)
, attrPath({alloc.allocate_object<AttrName>(attrPath.size()), attrPath.size()}) , attrPath({alloc.allocate_object<AttrName>(attrPath.size()), attrPath.size()})
{ {
@@ -395,13 +401,9 @@ struct ExprAttrs : Expr
} }
}; };
typedef std::pmr::map<Symbol, AttrDef> AttrDefs; typedef std::map<Symbol, AttrDef> AttrDefs;
/** AttrDefs attrs;
* attrs will never be null. we use std::optional so that we can call emplace() to re-initialize the value with a std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
* new pmr::map using a different allocator (move assignment will copy into the old allocator)
*/
std::optional<AttrDefs> attrs;
std::unique_ptr<std::pmr::vector<Expr *>> inheritFromExprs;
struct DynamicAttrDef struct DynamicAttrDef
{ {
@@ -413,20 +415,13 @@ struct ExprAttrs : Expr
, pos(pos) {}; , pos(pos) {};
}; };
typedef std::pmr::vector<DynamicAttrDef> DynamicAttrDefs; typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
/** DynamicAttrDefs dynamicAttrs;
* dynamicAttrs will never be null. See comment on AttrDefs above.
*/
std::optional<DynamicAttrDefs> dynamicAttrs;
ExprAttrs(const PosIdx & pos) ExprAttrs(const PosIdx & pos)
: recursive(false) : recursive(false)
, pos(pos) , pos(pos) {};
, attrs(AttrDefs{})
, dynamicAttrs(DynamicAttrDefs{}) {};
ExprAttrs() ExprAttrs()
: recursive(false) : recursive(false) {};
, attrs(AttrDefs{})
, dynamicAttrs(DynamicAttrDefs{}) {};
PosIdx getPos() const override PosIdx getPos() const override
{ {
@@ -444,7 +439,7 @@ struct ExprList : Expr
{ {
std::span<Expr *> elems; std::span<Expr *> elems;
ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::span<Expr *> exprs) ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::vector<Expr *> exprs)
: elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()}) : elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()})
{ {
std::ranges::copy(exprs, elems.begin()); std::ranges::copy(exprs, elems.begin());
@@ -573,7 +568,7 @@ public:
const PosTable & positions, const PosTable & positions,
std::pmr::polymorphic_allocator<char> & alloc, std::pmr::polymorphic_allocator<char> & alloc,
PosIdx pos, PosIdx pos,
const FormalsBuilder & formals, FormalsBuilder formals,
Expr * body) Expr * body)
: ExprLambda(positions, alloc, pos, Symbol(), formals, body) {}; : ExprLambda(positions, alloc, pos, Symbol(), formals, body) {};
@@ -592,14 +587,11 @@ public:
struct ExprCall : Expr struct ExprCall : Expr
{ {
Expr * fun; Expr * fun;
/** std::vector<Expr *> args;
* args will never be null. See comment on ExprAttrs::AttrDefs below.
*/
std::optional<std::pmr::vector<Expr *>> args;
PosIdx pos; PosIdx pos;
std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118 std::optional<PosIdx> cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118
ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args) ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
: fun(fun) : fun(fun)
, args(args) , args(args)
, pos(pos) , pos(pos)
@@ -607,7 +599,7 @@ struct ExprCall : Expr
{ {
} }
ExprCall(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos) ExprCall(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
: fun(fun) : fun(fun)
, args(args) , args(args)
, pos(pos) , pos(pos)
@@ -767,19 +759,7 @@ struct ExprConcatStrings : Expr
std::pmr::polymorphic_allocator<char> & alloc, std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos, const PosIdx & pos,
bool forceString, bool forceString,
std::span<std::pair<PosIdx, Expr *>> es) const std::vector<std::pair<PosIdx, Expr *>> & es)
: pos(pos)
, forceString(forceString)
, es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()})
{
std::ranges::copy(es, this->es.begin());
};
ExprConcatStrings(
std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
bool forceString,
std::initializer_list<std::pair<PosIdx, Expr *>> es)
: pos(pos) : pos(pos)
, forceString(forceString) , forceString(forceString)
, es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()}) , es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()})
@@ -839,7 +819,7 @@ public:
// we define some calls to add explicitly so that the argument can be passed in as initializer lists // we define some calls to add explicitly so that the argument can be passed in as initializer lists
template<class C> template<class C>
[[gnu::always_inline]] [[gnu::always_inline]]
C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args) C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args)
requires(std::same_as<C, ExprCall>) requires(std::same_as<C, ExprCall>)
{ {
return alloc.new_object<C>(pos, fun, std::move(args)); return alloc.new_object<C>(pos, fun, std::move(args));
@@ -847,7 +827,7 @@ public:
template<class C> template<class C>
[[gnu::always_inline]] [[gnu::always_inline]]
C * add(const PosIdx & pos, Expr * fun, std::pmr::vector<Expr *> && args, PosIdx && cursedOrEndPos) C * add(const PosIdx & pos, Expr * fun, std::vector<Expr *> && args, PosIdx && cursedOrEndPos)
requires(std::same_as<C, ExprCall>) requires(std::same_as<C, ExprCall>)
{ {
return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos)); return alloc.new_object<C>(pos, fun, std::move(args), std::move(cursedOrEndPos));
@@ -859,19 +839,7 @@ public:
add(std::pmr::polymorphic_allocator<char> & alloc, add(std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos, const PosIdx & pos,
bool forceString, bool forceString,
std::span<std::pair<PosIdx, Expr *>> es) const std::vector<std::pair<PosIdx, Expr *>> & es)
requires(std::same_as<C, ExprConcatStrings>)
{
return alloc.new_object<C>(alloc, pos, forceString, es);
}
template<class C>
[[gnu::always_inline]]
C *
add(std::pmr::polymorphic_allocator<char> & alloc,
const PosIdx & pos,
bool forceString,
std::initializer_list<std::pair<PosIdx, Expr *>> es)
requires(std::same_as<C, ExprConcatStrings>) requires(std::same_as<C, ExprConcatStrings>)
{ {
return alloc.new_object<C>(alloc, pos, forceString, es); return alloc.new_object<C>(alloc, pos, forceString, es);
+21 -23
View File
@@ -4,8 +4,6 @@
#include <limits> #include <limits>
#include "nix/expr/eval.hh" #include "nix/expr/eval.hh"
#include "nix/expr/value.hh"
#include "nix/expr/static-string-data.hh"
namespace nix { namespace nix {
@@ -96,7 +94,7 @@ struct ParserState
ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def); void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
void validateFormals(FormalsBuilder & formals, PosIdx pos = noPos, Symbol arg = {}); void validateFormals(FormalsBuilder & formals, PosIdx pos = noPos, Symbol arg = {});
Expr * stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std::variant<Expr *, StringToken>>> es); Expr * stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
PosIdx at(const ParserLocation & loc); PosIdx at(const ParserLocation & loc);
}; };
@@ -126,8 +124,8 @@ inline void ParserState::addAttr(
for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
ExprAttrs * nested; ExprAttrs * nested;
if (i->symbol) { if (i->symbol) {
ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(i->symbol); ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
if (j != attrs->attrs->end()) { if (j != attrs->attrs.end()) {
nested = dynamic_cast<ExprAttrs *>(j->second.e); nested = dynamic_cast<ExprAttrs *>(j->second.e);
if (!nested) { if (!nested) {
attrPath.erase(i + 1, attrPath.end()); attrPath.erase(i + 1, attrPath.end());
@@ -135,11 +133,11 @@ inline void ParserState::addAttr(
} }
} else { } else {
nested = exprs.add<ExprAttrs>(); nested = exprs.add<ExprAttrs>();
(*attrs->attrs)[i->symbol] = ExprAttrs::AttrDef(nested, pos); attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
} }
} else { } else {
nested = exprs.add<ExprAttrs>(); nested = exprs.add<ExprAttrs>();
attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
} }
attrs = nested; attrs = nested;
} }
@@ -148,7 +146,7 @@ inline void ParserState::addAttr(
if (i->symbol) { if (i->symbol) {
addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos)); addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
} else { } else {
attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
} }
auto it = lexerState.positionToDocComment.find(pos); auto it = lexerState.positionToDocComment.find(pos);
@@ -165,8 +163,8 @@ inline void ParserState::addAttr(
inline void inline void
ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def) ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
{ {
ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(symbol); ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
if (j != attrs->attrs->end()) { if (j != attrs->attrs.end()) {
// This attr path is already defined. However, if both // This attr path is already defined. However, if both
// e and the expr pointed by the attr path are two attribute sets, // e and the expr pointed by the attr path are two attribute sets,
// we want to merge them. // we want to merge them.
@@ -181,8 +179,8 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
// See https://github.com/NixOS/nix/issues/9020. // See https://github.com/NixOS/nix/issues/9020.
if (jAttrs && ae) { if (jAttrs && ae) {
if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
jAttrs->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(); jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
for (auto & ad : *ae->attrs) { for (auto & ad : ae->attrs) {
if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e); auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e); auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
@@ -192,12 +190,12 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
addAttr(jAttrs, attrPath, ad.first, std::move(ad.second)); addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
attrPath.pop_back(); attrPath.pop_back();
} }
ae->attrs->clear(); ae->attrs.clear();
jAttrs->dynamicAttrs->insert( jAttrs->dynamicAttrs.insert(
jAttrs->dynamicAttrs->end(), jAttrs->dynamicAttrs.end(),
std::make_move_iterator(ae->dynamicAttrs->begin()), std::make_move_iterator(ae->dynamicAttrs.begin()),
std::make_move_iterator(ae->dynamicAttrs->end())); std::make_move_iterator(ae->dynamicAttrs.end()));
ae->dynamicAttrs->clear(); ae->dynamicAttrs.clear();
if (ae->inheritFromExprs) { if (ae->inheritFromExprs) {
jAttrs->inheritFromExprs->insert( jAttrs->inheritFromExprs->insert(
jAttrs->inheritFromExprs->end(), jAttrs->inheritFromExprs->end(),
@@ -210,7 +208,7 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
} }
} else { } else {
// This attr path is not defined. Let's create it. // This attr path is not defined. Let's create it.
attrs->attrs->emplace(symbol, def); attrs->attrs.emplace(symbol, def);
def.e->setName(symbol); def.e->setName(symbol);
} }
} }
@@ -239,10 +237,10 @@ inline void ParserState::validateFormals(FormalsBuilder & formals, PosIdx pos, S
} }
inline Expr * inline Expr *
ParserState::stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std::variant<Expr *, StringToken>>> es) ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
{ {
if (es.empty()) if (es.empty())
return exprs.add<ExprString>(""_sds); return exprs.add<ExprString>("");
/* Figure out the minimum indentation. Note that by design /* Figure out the minimum indentation. Note that by design
whitespace-only final lines are not taken into account. (So whitespace-only final lines are not taken into account. (So
@@ -334,7 +332,7 @@ ParserState::stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std:
// If there is nothing at all, return the empty string directly. // If there is nothing at all, return the empty string directly.
// This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters. // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
if (es2.size() == 0) { if (es2.size() == 0) {
auto * const result = exprs.add<ExprString>(""_sds); auto * const result = exprs.add<ExprString>("");
return result; return result;
} }
@@ -343,7 +341,7 @@ ParserState::stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std:
auto * const result = (es2)[0].second; auto * const result = (es2)[0].second;
return result; return result;
} }
return exprs.add<ExprConcatStrings>(exprs.alloc, pos, true, es2); return exprs.add<ExprConcatStrings>(exprs.alloc, pos, true, std::move(es2));
} }
inline PosIdx LexerState::at(const ParserLocation & loc) inline PosIdx LexerState::at(const ParserLocation & loc)
+5 -1
View File
@@ -12,7 +12,11 @@ struct RegisterPrimOp
{ {
typedef std::vector<PrimOp> PrimOps; typedef std::vector<PrimOp> PrimOps;
static PrimOps & primOps(); static PrimOps & primOps()
{
static PrimOps primOps;
return primOps;
}
/** /**
* You can register a constant by passing an arity of 0. fun * You can register a constant by passing an arity of 0. fun
@@ -110,7 +110,7 @@ struct PrintOptions
* `PrintOptions` for unknown and therefore potentially large values in error messages, * `PrintOptions` for unknown and therefore potentially large values in error messages,
* to avoid printing "too much" output. * to avoid printing "too much" output.
*/ */
static constexpr PrintOptions errorPrintOptions = PrintOptions{ static PrintOptions errorPrintOptions = PrintOptions{
.ansiColors = true, .ansiColors = true,
.maxDepth = 10, .maxDepth = 10,
.maxAttrs = 10, .maxAttrs = 10,
@@ -1,44 +0,0 @@
#pragma once
///@file
#include "nix/expr/value.hh"
namespace nix {
template<size_t N>
struct StringData::Static
{
/**
* @note Must be first to make layout compatible with StringData.
*/
const size_t size = N - 1;
char data[N];
consteval Static(const char (&str)[N])
{
static_assert(N > 0);
if (str[size] != '\0')
throw;
std::copy_n(str, N, data);
}
operator const StringData &() const &
{
static_assert(sizeof(decltype(*this)) >= sizeof(StringData));
static_assert(alignof(decltype(*this)) == alignof(StringData));
/* NOTE: This cast is somewhat on the fence of what's legal in C++.
The question boils down to whether flexible array members are
layout compatible with fixed-size arrays. This is a gray area, since
FAMs are not standard anyway.
*/
return *reinterpret_cast<const StringData *>(this);
}
};
template<StringData::Static S>
const StringData & operator""_sds()
{
return S;
}
} // namespace nix
+15 -20
View File
@@ -3,7 +3,6 @@
#include <memory_resource> #include <memory_resource>
#include "nix/expr/value.hh" #include "nix/expr/value.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/util/chunked-vector.hh" #include "nix/util/chunked-vector.hh"
#include "nix/util/error.hh" #include "nix/util/error.hh"
@@ -17,6 +16,7 @@ class SymbolValue : protected Value
friend class SymbolStr; friend class SymbolStr;
friend class SymbolTable; friend class SymbolTable;
uint32_t size_;
uint32_t idx; uint32_t idx;
SymbolValue() = default; SymbolValue() = default;
@@ -24,7 +24,7 @@ class SymbolValue : protected Value
public: public:
operator std::string_view() const noexcept operator std::string_view() const noexcept
{ {
return string_view(); return {c_str(), size_};
} }
}; };
@@ -96,13 +96,13 @@ class SymbolStr
SymbolValueStore & store; SymbolValueStore & store;
std::string_view s; std::string_view s;
std::size_t hash; std::size_t hash;
std::pmr::memory_resource & resource; std::pmr::polymorphic_allocator<char> & alloc;
Key(SymbolValueStore & store, std::string_view s, std::pmr::memory_resource & stringMemory) Key(SymbolValueStore & store, std::string_view s, std::pmr::polymorphic_allocator<char> & stringAlloc)
: store(store) : store(store)
, s(s) , s(s)
, hash(HashType{}(s)) , hash(HashType{}(s))
, resource(stringMemory) , alloc(stringAlloc)
{ {
} }
}; };
@@ -122,10 +122,14 @@ public:
// for multi-threaded implementations: lock store and allocator here // for multi-threaded implementations: lock store and allocator here
const auto & [v, idx] = key.store.add(SymbolValue{}); const auto & [v, idx] = key.store.add(SymbolValue{});
if (size == 0) { if (size == 0) {
v.mkStringNoCopy(""_sds, nullptr); v.mkStringNoCopy("", nullptr);
} else { } else {
v.mkStringNoCopy(StringData::make(key.resource, key.s)); auto s = key.alloc.allocate(size + 1);
memcpy(s, key.s.data(), size);
s[size] = '\0';
v.mkStringNoCopy(s, nullptr);
} }
v.size_ = size;
v.idx = idx; v.idx = idx;
this->s = &v; this->s = &v;
} }
@@ -135,12 +139,6 @@ public:
return *s == s2; return *s == s2;
} }
[[gnu::always_inline]]
const StringData & string_data() const noexcept
{
return s->string_data();
}
[[gnu::always_inline]] [[gnu::always_inline]]
const char * c_str() const noexcept const char * c_str() const noexcept
{ {
@@ -157,17 +155,13 @@ public:
[[gnu::always_inline]] [[gnu::always_inline]]
bool empty() const noexcept bool empty() const noexcept
{ {
auto * p = &s->string_data(); return s->size_ == 0;
// Save a dereference in the sentinel value case
if (p == &""_sds)
return true;
return p->size() == 0;
} }
[[gnu::always_inline]] [[gnu::always_inline]]
size_t size() const noexcept size_t size() const noexcept
{ {
return s->string_data().size(); return s->size_;
} }
[[gnu::always_inline]] [[gnu::always_inline]]
@@ -265,6 +259,7 @@ private:
* During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based. * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based.
*/ */
std::pmr::monotonic_buffer_resource buffer; std::pmr::monotonic_buffer_resource buffer;
std::pmr::polymorphic_allocator<char> stringAlloc{&buffer};
SymbolStr::SymbolValueStore store{16}; SymbolStr::SymbolValueStore store{16};
/** /**
@@ -287,7 +282,7 @@ public:
// Most symbols are looked up more than once, so we trade off insertion performance // Most symbols are looked up more than once, so we trade off insertion performance
// for lookup performance. // for lookup performance.
// FIXME: make this thread-safe. // FIXME: make this thread-safe.
return Symbol(*symbols.insert(SymbolStr::Key{store, s, buffer}).first); return Symbol(*symbols.insert(SymbolStr::Key{store, s, stringAlloc}).first);
} }
std::vector<SymbolStr> resolve(const std::vector<Symbol> & symbols) const std::vector<SymbolStr> resolve(const std::vector<Symbol> & symbols) const
+25 -123
View File
@@ -1,14 +1,8 @@
#pragma once #pragma once
///@file ///@file
#include <bit>
#include <cassert> #include <cassert>
#include <cstddef>
#include <cstring>
#include <memory>
#include <memory_resource>
#include <span> #include <span>
#include <string_view>
#include <type_traits> #include <type_traits>
#include <concepts> #include <concepts>
@@ -88,7 +82,6 @@ class PosIdx;
struct Pos; struct Pos;
class StorePath; class StorePath;
class EvalState; class EvalState;
class EvalMemory;
class XMLWriter; class XMLWriter;
class Printer; class Printer;
@@ -162,7 +155,7 @@ class ListBuilder
Value * inlineElems[2] = {nullptr, nullptr}; Value * inlineElems[2] = {nullptr, nullptr};
public: public:
Value ** elems; Value ** elems;
ListBuilder(EvalMemory & mem, size_t size); ListBuilder(size_t size);
// NOTE: Can be noexcept because we are just copying integral values and // NOTE: Can be noexcept because we are just copying integral values and
// raw pointers. // raw pointers.
@@ -193,91 +186,6 @@ public:
friend struct Value; friend struct Value;
}; };
class StringData
{
public:
using size_type = std::size_t;
size_type size_;
char data_[];
/*
* This in particular ensures that we cannot have a `StringData`
* that we use by value, which is just what we want!
*
* Dynamically sized types aren't a thing in C++ and even flexible array
* members are a language extension and beyond the realm of standard C++.
* Technically, sizeof data_ member is 0 and the intended way to use flexible
* array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
* and the compiler will consider alignment restrictions for the FAM.
*
*/
StringData(StringData &&) = delete;
StringData & operator=(StringData &&) = delete;
StringData(const StringData &) = delete;
StringData & operator=(const StringData &) = delete;
~StringData() = default;
private:
StringData() = delete;
explicit StringData(size_type size)
: size_(size)
{
}
public:
/**
* Allocate StringData on the (possibly) GC-managed heap and copy
* the contents of s to it.
*/
static const StringData & make(EvalMemory & mem, std::string_view s);
/**
* Allocate StringData on the (possibly) GC-managed heap.
* @param size Length of the string (without the NUL terminator).
*/
static StringData & alloc(EvalMemory & mem, size_t size);
size_t size() const
{
return size_;
}
char * data() noexcept
{
return data_;
}
const char * data() const noexcept
{
return data_;
}
const char * c_str() const noexcept
{
return data_;
}
constexpr std::string_view view() const noexcept
{
return std::string_view(data_, size_);
}
template<size_t N>
struct Static;
static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
{
auto & res =
*new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
std::memcpy(res.data_, s.data(), s.size());
res.data_[s.size()] = '\0';
return res;
}
};
namespace detail { namespace detail {
/** /**
@@ -311,7 +219,7 @@ struct ValueBase
*/ */
struct StringWithContext struct StringWithContext
{ {
const StringData * str; const char * c_str;
/** /**
* The type of the context itself. * The type of the context itself.
@@ -326,7 +234,7 @@ struct ValueBase
*/ */
struct Context struct Context
{ {
using value_type = const StringData *; using value_type = const char *;
using size_type = std::size_t; using size_type = std::size_t;
using iterator = const value_type *; using iterator = const value_type *;
@@ -365,7 +273,7 @@ struct ValueBase
/** /**
* @return null pointer when context.empty() * @return null pointer when context.empty()
*/ */
static Context * fromBuilder(const NixStringContext & context, EvalMemory & mem); static Context * fromBuilder(const NixStringContext & context);
}; };
/** /**
@@ -377,7 +285,7 @@ struct ValueBase
struct Path struct Path
{ {
SourceAccessor * accessor; SourceAccessor * accessor;
const StringData * path; const char * path;
}; };
struct Null struct Null
@@ -738,13 +646,13 @@ protected:
void getStorage(StringWithContext & string) const noexcept void getStorage(StringWithContext & string) const noexcept
{ {
string.context = untagPointer<decltype(string.context)>(payload[0]); string.context = untagPointer<decltype(string.context)>(payload[0]);
string.str = std::bit_cast<const StringData *>(payload[1]); string.c_str = std::bit_cast<const char *>(payload[1]);
} }
void getStorage(Path & path) const noexcept void getStorage(Path & path) const noexcept
{ {
path.accessor = untagPointer<decltype(path.accessor)>(payload[0]); path.accessor = untagPointer<decltype(path.accessor)>(payload[0]);
path.path = std::bit_cast<const StringData *>(payload[1]); path.path = std::bit_cast<const char *>(payload[1]);
} }
void setStorage(NixInt integer) noexcept void setStorage(NixInt integer) noexcept
@@ -789,7 +697,7 @@ protected:
void setStorage(StringWithContext string) noexcept void setStorage(StringWithContext string) noexcept
{ {
setUntaggablePayload<pdString>(string.context, string.str); setUntaggablePayload<pdString>(string.context, string.c_str);
} }
void setStorage(Path path) noexcept void setStorage(Path path) noexcept
@@ -1142,22 +1050,22 @@ public:
setStorage(b); setStorage(b);
} }
void mkStringNoCopy(const StringData & s, const Value::StringWithContext::Context * context = nullptr) noexcept void mkStringNoCopy(const char * s, const Value::StringWithContext::Context * context = nullptr) noexcept
{ {
setStorage(StringWithContext{.str = &s, .context = context}); setStorage(StringWithContext{.c_str = s, .context = context});
} }
void mkString(std::string_view s, EvalMemory & mem); void mkString(std::string_view s);
void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem); void mkString(std::string_view s, const NixStringContext & context);
void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem); void mkStringMove(const char * s, const NixStringContext & context);
void mkPath(const SourcePath & path, EvalMemory & mem); void mkPath(const SourcePath & path);
inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept inline void mkPath(SourceAccessor * accessor, const char * path) noexcept
{ {
setStorage(Path{.accessor = accessor, .path = &path}); setStorage(Path{.accessor = accessor, .path = path});
} }
inline void mkNull() noexcept inline void mkNull() noexcept
@@ -1255,23 +1163,17 @@ public:
SourcePath path() const SourcePath path() const
{ {
return SourcePath( return SourcePath(ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), pathStr()));
ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), std::string(pathStrView())));
}
const StringData & string_data() const noexcept
{
return *getStorage<StringWithContext>().str;
}
const char * c_str() const noexcept
{
return getStorage<StringWithContext>().str->data();
} }
std::string_view string_view() const noexcept std::string_view string_view() const noexcept
{ {
return string_data().view(); return std::string_view{getStorage<StringWithContext>().c_str};
}
const char * c_str() const noexcept
{
return getStorage<StringWithContext>().c_str;
} }
const Value::StringWithContext::Context * context() const noexcept const Value::StringWithContext::Context * context() const noexcept
@@ -1331,12 +1233,12 @@ public:
const char * pathStr() const noexcept const char * pathStr() const noexcept
{ {
return getStorage<Path>().path->c_str(); return getStorage<Path>().path;
} }
std::string_view pathStrView() const noexcept std::string_view pathStrView() const noexcept
{ {
return getStorage<Path>().path->view(); return std::string_view{getStorage<Path>().path};
} }
SourceAccessor * pathAccessor() const noexcept SourceAccessor * pathAccessor() const noexcept
+1 -1
View File
@@ -151,7 +151,7 @@ public:
bool string(string_t & val) override bool string(string_t & val) override
{ {
forceNoNullByte(val); forceNoNullByte(val);
rs->value(state).mkString(val, state.mem); rs->value(state).mkString(val);
rs->add(); rs->add();
return true; return true;
} }
+14 -31
View File
@@ -74,9 +74,9 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
{ {
typedef const AttrDefs::value_type * Attr; typedef const decltype(attrs)::value_type * Attr;
std::vector<Attr> sorted; std::vector<Attr> sorted;
for (auto & i : *attrs) for (auto & i : attrs)
sorted.push_back(&i); sorted.push_back(&i);
std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) { std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
std::string_view sa = symbols[a->first], sb = symbols[b->first]; std::string_view sa = symbols[a->first], sb = symbols[b->first];
@@ -122,7 +122,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
str << "; "; str << "; ";
} }
} }
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
str << "\"${"; str << "\"${";
i.nameExpr->show(symbols, str); i.nameExpr->show(symbols, str);
str << "}\" = "; str << "}\" = ";
@@ -191,7 +191,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const
{ {
str << '('; str << '(';
fun->show(symbols, str); fun->show(symbols, str);
for (auto e : *args) { for (auto e : args) {
str << ' '; str << ' ';
e->show(symbols, str); e->show(symbols, str);
} }
@@ -401,26 +401,15 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
AttrDefs newAttrs{std::move(*attrs), arena};
attrs.emplace(std::move(newAttrs), arena);
DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
if (inheritFromExprs)
inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
}
if (es.debugRepl) if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env)); es.exprEnvs.insert(std::make_pair(this, env));
if (recursive) { if (recursive) {
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> { auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->size()); auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs.size());
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs) for (auto & i : attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++); newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv; return newEnv;
}(); }();
@@ -428,20 +417,20 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
// No need to sort newEnv since attrs is in sorted order. // No need to sort newEnv since attrs is in sorted order.
auto inheritFromEnv = bindInheritSources(es, newEnv); auto inheritFromEnv = bindInheritSources(es, newEnv);
for (auto & i : *attrs) for (auto & i : attrs)
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, newEnv); i.nameExpr->bindVars(es, newEnv);
i.valueExpr->bindVars(es, newEnv); i.valueExpr->bindVars(es, newEnv);
} }
} else { } else {
auto inheritFromEnv = bindInheritSources(es, env); auto inheritFromEnv = bindInheritSources(es, env);
for (auto & i : *attrs) for (auto & i : attrs)
i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
for (auto & i : *dynamicAttrs) { for (auto & i : dynamicAttrs) {
i.nameExpr->bindVars(es, env); i.nameExpr->bindVars(es, env);
i.valueExpr->bindVars(es, env); i.valueExpr->bindVars(es, env);
} }
@@ -486,27 +475,21 @@ void ExprLambda::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
// Move storage into the Exprs arena
{
auto arena = es.mem.exprs.alloc;
std::pmr::vector<Expr *> newArgs{std::move(*args), arena};
args.emplace(std::move(newArgs), arena);
}
if (es.debugRepl) if (es.debugRepl)
es.exprEnvs.insert(std::make_pair(this, env)); es.exprEnvs.insert(std::make_pair(this, env));
fun->bindVars(es, env); fun->bindVars(es, env);
for (auto e : *args) for (auto e : args)
e->bindVars(es, env); e->bindVars(es, env);
} }
void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
{ {
auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> { auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size()); auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs.size());
Displacement displ = 0; Displacement displ = 0;
for (auto & i : *attrs->attrs) for (auto & i : attrs->attrs)
newEnv->vars.emplace_back(i.first, i.second.displ = displ++); newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
return newEnv; return newEnv;
}(); }();
@@ -514,7 +497,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
// No need to sort newEnv since attrs->attrs is in sorted order. // No need to sort newEnv since attrs->attrs is in sorted order.
auto inheritFromEnv = attrs->bindInheritSources(es, newEnv); auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
for (auto & i : *attrs->attrs) for (auto & i : attrs->attrs)
i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
if (es.debugRepl) if (es.debugRepl)
+5
View File
@@ -70,6 +70,11 @@ mkMesonLibrary (finalAttrs: {
nix-util nix-util
nix-store nix-store
nix-fetchers nix-fetchers
]
++ finalAttrs.passthru.externalPropagatedBuildInputs;
# Hack for sake of the dev shell
passthru.externalPropagatedBuildInputs = [
boost boost
nlohmann_json nlohmann_json
] ]
+40 -40
View File
@@ -115,7 +115,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P
static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) { static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
if (auto e2 = dynamic_cast<ExprCall *>(fn)) { if (auto e2 = dynamic_cast<ExprCall *>(fn)) {
e2->args->push_back(arg); e2->args.push_back(arg);
return fn; return fn;
} }
return exprs.add<ExprCall>(pos, fn, {arg}); return exprs.add<ExprCall>(pos, fn, {arg});
@@ -126,26 +126,26 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
%define api.value.type variant %define api.value.type variant
%type <Expr *> start expr expr_function expr_if expr_op %type <nix::Expr *> start expr expr_function expr_if expr_op
%type <Expr *> expr_select expr_simple expr_app %type <nix::Expr *> expr_select expr_simple expr_app
%type <Expr *> expr_pipe_from expr_pipe_into %type <nix::Expr *> expr_pipe_from expr_pipe_into
%type <std::pmr::vector<Expr *>> list %type <std::vector<Expr *>> list
%type <ExprAttrs *> binds binds1 %type <nix::ExprAttrs *> binds binds1
%type <FormalsBuilder> formals formal_set %type <nix::FormalsBuilder> formals formal_set
%type <Formal> formal %type <nix::Formal> formal
%type <std::vector<AttrName>> attrpath %type <std::vector<nix::AttrName>> attrpath
%type <std::vector<std::pair<AttrName, PosIdx>>> attrs %type <std::vector<std::pair<nix::AttrName, nix::PosIdx>>> attrs
%type <std::vector<std::pair<PosIdx, Expr *>>> string_parts_interpolated %type <std::vector<std::pair<nix::PosIdx, nix::Expr *>>> string_parts_interpolated
%type <std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>> ind_string_parts %type <std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>>> ind_string_parts
%type <Expr *> path_start %type <nix::Expr *> path_start
%type <std::variant<Expr *, std::string_view>> string_parts string_attr %type <std::variant<nix::Expr *, std::string_view>> string_parts string_attr
%type <StringToken> attr %type <nix::StringToken> attr
%token <StringToken> ID %token <nix::StringToken> ID
%token <StringToken> STR IND_STR %token <nix::StringToken> STR IND_STR
%token <NixInt> INT_LIT %token <nix::NixInt> INT_LIT
%token <NixFloat> FLOAT_LIT %token <nix::NixFloat> FLOAT_LIT
%token <StringToken> PATH HPATH SPATH PATH_END %token <nix::StringToken> PATH HPATH SPATH PATH_END
%token <StringToken> URI %token <nix::StringToken> URI
%token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
%token PIPE_FROM PIPE_INTO /* <| and |> */ %token PIPE_FROM PIPE_INTO /* <| and |> */
%token DOLLAR_CURLY /* == ${ */ %token DOLLAR_CURLY /* == ${ */
@@ -186,7 +186,7 @@ expr_function
| formal_set ':' expr_function[body] | formal_set ':' expr_function[body]
{ {
state->validateFormals($formal_set); state->validateFormals($formal_set);
auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, $formal_set, $body); auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, std::move($formal_set), $body);
$$ = me; $$ = me;
SET_DOC_POS(me, @1); SET_DOC_POS(me, @1);
} }
@@ -194,7 +194,7 @@ expr_function
{ {
auto arg = state->symbols.create($ID); auto arg = state->symbols.create($ID);
state->validateFormals($formal_set, CUR_POS, arg); state->validateFormals($formal_set, CUR_POS, arg);
auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, $formal_set, $body); auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, std::move($formal_set), $body);
$$ = me; $$ = me;
SET_DOC_POS(me, @1); SET_DOC_POS(me, @1);
} }
@@ -202,7 +202,7 @@ expr_function
{ {
auto arg = state->symbols.create($ID); auto arg = state->symbols.create($ID);
state->validateFormals($formal_set, CUR_POS, arg); state->validateFormals($formal_set, CUR_POS, arg);
auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, $formal_set, $body); auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, std::move($formal_set), $body);
$$ = me; $$ = me;
SET_DOC_POS(me, @1); SET_DOC_POS(me, @1);
} }
@@ -211,7 +211,7 @@ expr_function
| WITH expr ';' expr_function | WITH expr ';' expr_function
{ $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); } { $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); }
| LET binds IN_KW expr_function | LET binds IN_KW expr_function
{ if (!$2->dynamicAttrs->empty()) { if (!$2->dynamicAttrs.empty())
throw ParseError({ throw ParseError({
.msg = HintFmt("dynamic attributes not allowed in let"), .msg = HintFmt("dynamic attributes not allowed in let"),
.pos = state->positions[CUR_POS] .pos = state->positions[CUR_POS]
@@ -251,7 +251,7 @@ expr_op
| expr_op OR expr_op { $$ = state->exprs.add<ExprOpOr>(state->at(@2), $1, $3); } | expr_op OR expr_op { $$ = state->exprs.add<ExprOpOr>(state->at(@2), $1, $3); }
| expr_op IMPL expr_op { $$ = state->exprs.add<ExprOpImpl>(state->at(@2), $1, $3); } | expr_op IMPL expr_op { $$ = state->exprs.add<ExprOpImpl>(state->at(@2), $1, $3); }
| expr_op UPDATE expr_op { $$ = state->exprs.add<ExprOpUpdate>(state->at(@2), $1, $3); } | expr_op UPDATE expr_op { $$ = state->exprs.add<ExprOpUpdate>(state->at(@2), $1, $3); }
| expr_op '?' attrpath { $$ = state->exprs.add<ExprOpHasAttr>(state->exprs.alloc, $1, $3); } | expr_op '?' attrpath { $$ = state->exprs.add<ExprOpHasAttr>(state->exprs.alloc, $1, std::move($3)); }
| expr_op '+' expr_op | expr_op '+' expr_op
{ $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, state->at(@2), false, {{state->at(@1), $1}, {state->at(@3), $3}}); } { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, state->at(@2), false, {{state->at(@1), $1}, {state->at(@3), $3}}); }
| expr_op '-' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.sub), {$1, $3}); } | expr_op '-' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.sub), {$1, $3}); }
@@ -272,9 +272,9 @@ expr_app
expr_select expr_select
: expr_simple '.' attrpath : expr_simple '.' attrpath
{ $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, $3, nullptr); } { $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, std::move($3), nullptr); }
| expr_simple '.' attrpath OR_KW expr_select | expr_simple '.' attrpath OR_KW expr_select
{ $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, $3, $5); $5->warnIfCursedOr(state->symbols, state->positions); } { $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, std::move($3), $5); $5->warnIfCursedOr(state->symbols, state->positions); }
| /* Backwards compatibility: because Nixpkgs has a function named ‘or’, | /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
allow stuff like ‘map or [...]’. This production is problematic (see allow stuff like ‘map or [...]’. This production is problematic (see
https://github.com/NixOS/nix/issues/11118) and will be refactored in the https://github.com/NixOS/nix/issues/11118) and will be refactored in the
@@ -304,12 +304,12 @@ expr_simple
$2); $2);
} }
| IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
$$ = state->stripIndentation(CUR_POS, $2); $$ = state->stripIndentation(CUR_POS, std::move($2));
} }
| path_start PATH_END | path_start PATH_END
| path_start string_parts_interpolated PATH_END { | path_start string_parts_interpolated PATH_END {
$2.insert($2.begin(), {state->at(@1), $1}); $2.insert($2.begin(), {state->at(@1), $1});
$$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, false, $2); $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, false, std::move($2));
} }
| SPATH { | SPATH {
std::string_view path($1.p + 1, $1.l - 2); std::string_view path($1.p + 1, $1.l - 2);
@@ -338,12 +338,12 @@ expr_simple
{ $2->pos = CUR_POS; $$ = $2; } { $2->pos = CUR_POS; $$ = $2; }
| '{' '}' | '{' '}'
{ $$ = state->exprs.add<ExprAttrs>(CUR_POS); } { $$ = state->exprs.add<ExprAttrs>(CUR_POS); }
| '[' list ']' { $$ = state->exprs.add<ExprList>(state->exprs.alloc, $2); } | '[' list ']' { $$ = state->exprs.add<ExprList>(state->exprs.alloc, std::move($2)); }
; ;
string_parts string_parts
: STR { $$ = $1; } : STR { $$ = $1; }
| string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1); } | string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, std::move($1)); }
| { $$ = std::string_view(); } | { $$ = std::string_view(); }
; ;
@@ -413,9 +413,9 @@ binds1
| binds[accum] INHERIT attrs ';' | binds[accum] INHERIT attrs ';'
{ $$ = $accum; { $$ = $accum;
for (auto & [i, iPos] : $attrs) { for (auto & [i, iPos] : $attrs) {
if ($accum->attrs->find(i.symbol) != $accum->attrs->end()) if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos); state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs->emplace( $accum->attrs.emplace(
i.symbol, i.symbol,
ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
} }
@@ -423,13 +423,13 @@ binds1
| binds[accum] INHERIT '(' expr ')' attrs ';' | binds[accum] INHERIT '(' expr ')' attrs ';'
{ $$ = $accum; { $$ = $accum;
if (!$accum->inheritFromExprs) if (!$accum->inheritFromExprs)
$accum->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(); $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
$accum->inheritFromExprs->push_back($expr); $accum->inheritFromExprs->push_back($expr);
auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1); auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
for (auto & [i, iPos] : $attrs) { for (auto & [i, iPos] : $attrs) {
if ($accum->attrs->find(i.symbol) != $accum->attrs->end()) if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos); state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
$accum->attrs->emplace( $accum->attrs.emplace(
i.symbol, i.symbol,
ExprAttrs::AttrDef( ExprAttrs::AttrDef(
state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol), state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol),
+51 -63
View File
@@ -5,7 +5,6 @@
#include "nix/expr/eval-settings.hh" #include "nix/expr/eval-settings.hh"
#include "nix/expr/gc-small-vector.hh" #include "nix/expr/gc-small-vector.hh"
#include "nix/expr/json-to-value.hh" #include "nix/expr/json-to-value.hh"
#include "nix/expr/static-string-data.hh"
#include "nix/store/globals.hh" #include "nix/store/globals.hh"
#include "nix/store/names.hh" #include "nix/store/names.hh"
#include "nix/store/path-references.hh" #include "nix/store/path-references.hh"
@@ -40,12 +39,6 @@
namespace nix { namespace nix {
RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
{
static RegisterPrimOp::PrimOps primOps;
return primOps;
}
/************************************************************* /*************************************************************
* Miscellaneous * Miscellaneous
*************************************************************/ *************************************************************/
@@ -53,7 +46,7 @@ RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
static inline Value * mkString(EvalState & state, const std::csub_match & match) static inline Value * mkString(EvalState & state, const std::csub_match & match)
{ {
Value * v = state.allocValue(); Value * v = state.allocValue();
v->mkString({match.first, match.second}, state.mem); v->mkString({match.first, match.second});
return v; return v;
} }
@@ -228,14 +221,13 @@ void derivationToValue(
path2, path2,
{ {
NixStringContextElem::DrvDeep{.drvPath = storePath}, NixStringContextElem::DrvDeep{.drvPath = storePath},
}, });
state.mem); attrs.alloc(state.s.name).mkString(drv.env["name"]);
attrs.alloc(state.s.name).mkString(drv.env["name"], state.mem);
auto list = state.buildList(drv.outputs.size()); auto list = state.buildList(drv.outputs.size());
for (const auto & [i, o] : enumerate(drv.outputs)) { for (const auto & [i, o] : enumerate(drv.outputs)) {
mkOutputString(state, attrs, storePath, o); mkOutputString(state, attrs, storePath, o);
(list[i] = state.allocValue())->mkString(o.first, state.mem); (list[i] = state.allocValue())->mkString(o.first);
} }
attrs.alloc(state.s.outputs).mkList(list); attrs.alloc(state.s.outputs).mkList(list);
@@ -495,34 +487,34 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
state.forceValue(*args[0], pos); state.forceValue(*args[0], pos);
switch (args[0]->type()) { switch (args[0]->type()) {
case nInt: case nInt:
v.mkStringNoCopy("int"_sds); v.mkStringNoCopy("int");
break; break;
case nBool: case nBool:
v.mkStringNoCopy("bool"_sds); v.mkStringNoCopy("bool");
break; break;
case nString: case nString:
v.mkStringNoCopy("string"_sds); v.mkStringNoCopy("string");
break; break;
case nPath: case nPath:
v.mkStringNoCopy("path"_sds); v.mkStringNoCopy("path");
break; break;
case nNull: case nNull:
v.mkStringNoCopy("null"_sds); v.mkStringNoCopy("null");
break; break;
case nAttrs: case nAttrs:
v.mkStringNoCopy("set"_sds); v.mkStringNoCopy("set");
break; break;
case nList: case nList:
v.mkStringNoCopy("list"_sds); v.mkStringNoCopy("list");
break; break;
case nFunction: case nFunction:
v.mkStringNoCopy("lambda"_sds); v.mkStringNoCopy("lambda");
break; break;
case nExternal: case nExternal:
v.mkString(args[0]->external()->typeOf(), state.mem); v.mkString(args[0]->external()->typeOf());
break; break;
case nFloat: case nFloat:
v.mkStringNoCopy("float"_sds); v.mkStringNoCopy("float");
break; break;
case nThunk: case nThunk:
unreachable(); unreachable();
@@ -1176,7 +1168,7 @@ static void prim_getEnv(EvalState & state, const PosIdx pos, Value ** args, Valu
{ {
std::string name( std::string name(
state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv"));
v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""), state.mem); v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or(""));
} }
static RegisterPrimOp primop_getEnv({ static RegisterPrimOp primop_getEnv({
@@ -1818,8 +1810,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
drvPathS, drvPathS,
{ {
NixStringContextElem::DrvDeep{.drvPath = drvPath}, NixStringContextElem::DrvDeep{.drvPath = drvPath},
}, });
state.mem);
for (auto & i : drv.outputs) for (auto & i : drv.outputs)
mkOutputString(state, result, drvPath, i); mkOutputString(state, result, drvPath, i);
@@ -1842,10 +1833,8 @@ static RegisterPrimOp primop_derivationStrict(
‘out’. */ ‘out’. */
static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v) static void prim_placeholder(EvalState & state, const PosIdx pos, Value ** args, Value & v)
{ {
v.mkString( v.mkString(hashPlaceholder(
hashPlaceholder(state.forceStringNoCtx( state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")));
*args[0], pos, "while evaluating the first argument passed to builtins.placeholder")),
state.mem);
} }
static RegisterPrimOp primop_placeholder({ static RegisterPrimOp primop_placeholder({
@@ -1874,7 +1863,7 @@ static void prim_toPath(EvalState & state, const PosIdx pos, Value ** args, Valu
NixStringContext context; NixStringContext context;
auto path = auto path =
state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath");
v.mkString(path.path.abs(), context, state.mem); v.mkString(path.path.abs(), context);
} }
static RegisterPrimOp primop_toPath({ static RegisterPrimOp primop_toPath({
@@ -1917,7 +1906,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V
if (!settings.readOnlyMode) if (!settings.readOnlyMode)
state.store->ensurePath(path2); state.store->ensurePath(path2);
context.insert(NixStringContextElem::Opaque{.path = path2}); context.insert(NixStringContextElem::Opaque{.path = path2});
v.mkString(path.abs(), context, state.mem); v.mkString(path.abs(), context);
} }
static RegisterPrimOp primop_storePath({ static RegisterPrimOp primop_storePath({
@@ -1999,8 +1988,7 @@ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value ** args,
v.mkString( v.mkString(
legacyBaseNameOf(*state.coerceToString( legacyBaseNameOf(*state.coerceToString(
pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)), pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)),
context, context);
state.mem);
} }
static RegisterPrimOp primop_baseNameOf({ static RegisterPrimOp primop_baseNameOf({
@@ -2029,18 +2017,18 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
state.forceValue(*args[0], pos); state.forceValue(*args[0], pos);
if (args[0]->type() == nPath) { if (args[0]->type() == nPath) {
auto path = args[0]->path(); auto path = args[0]->path();
v.mkPath(path.path.isRoot() ? path : path.parent(), state.mem); v.mkPath(path.path.isRoot() ? path : path.parent());
} else { } else {
NixStringContext context; NixStringContext context;
auto path = state.coerceToString( auto path = state.coerceToString(
pos, *args[0], context, "while evaluating the first argument passed to 'builtins.dirOf'", false, false); pos, *args[0], context, "while evaluating the first argument passed to 'builtins.dirOf'", false, false);
auto pos = path->rfind('/'); auto pos = path->rfind('/');
if (pos == path->npos) if (pos == path->npos)
v.mkStringMove("."_sds, context, state.mem); v.mkStringMove(".", context);
else if (pos == 0) else if (pos == 0)
v.mkStringMove("/"_sds, context, state.mem); v.mkStringMove("/", context);
else else
v.mkString(path->substr(0, pos), context, state.mem); v.mkString(path->substr(0, pos), context);
} }
} }
@@ -2082,7 +2070,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Va
.path = std::move((StorePath &&) p), .path = std::move((StorePath &&) p),
}); });
} }
v.mkString(s, context, state.mem); v.mkString(s, context);
} }
static RegisterPrimOp primop_readFile({ static RegisterPrimOp primop_readFile({
@@ -2146,7 +2134,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va
auto path = auto path =
state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile");
v.mkPath(state.findFile(lookupPath, path, pos), state.mem); v.mkPath(state.findFile(lookupPath, path, pos));
} }
static RegisterPrimOp primop_findFile( static RegisterPrimOp primop_findFile(
@@ -2295,7 +2283,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value ** args, Va
auto path = realisePath(state, pos, *args[1]); auto path = realisePath(state, pos, *args[1]);
v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false), state.mem); v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false));
} }
static RegisterPrimOp primop_hashFile({ static RegisterPrimOp primop_hashFile({
@@ -2321,10 +2309,10 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
static const Constants stringValues = []() { static const Constants stringValues = []() {
Constants res; Constants res;
res.regular.mkStringNoCopy("regular"_sds); res.regular.mkStringNoCopy("regular");
res.directory.mkStringNoCopy("directory"_sds); res.directory.mkStringNoCopy("directory");
res.symlink.mkStringNoCopy("symlink"_sds); res.symlink.mkStringNoCopy("symlink");
res.unknown.mkStringNoCopy("unknown"_sds); res.unknown.mkStringNoCopy("unknown");
return res; return res;
}(); }();
@@ -2384,7 +2372,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val
// detailed node info quickly in this case we produce a thunk to // detailed node info quickly in this case we produce a thunk to
// query the file type lazily. // query the file type lazily.
auto epath = state.allocValue(); auto epath = state.allocValue();
epath->mkPath(path / name, state.mem); epath->mkPath(path / name);
if (!readFileType) if (!readFileType)
readFileType = &state.getBuiltin("readFileType"); readFileType = &state.getBuiltin("readFileType");
attr.mkApp(readFileType, epath); attr.mkApp(readFileType, epath);
@@ -2478,7 +2466,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value
std::ostringstream out; std::ostringstream out;
NixStringContext context; NixStringContext context;
printValueAsXML(state, true, false, *args[0], out, context, pos); printValueAsXML(state, true, false, *args[0], out, context, pos);
v.mkString(out.view(), context, state.mem); v.mkString(out.view(), context);
} }
static RegisterPrimOp primop_toXML({ static RegisterPrimOp primop_toXML({
@@ -2586,7 +2574,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu
std::ostringstream out; std::ostringstream out;
NixStringContext context; NixStringContext context;
printValueAsJSON(state, true, *args[0], pos, out, context); printValueAsJSON(state, true, *args[0], pos, out, context);
v.mkString(out.view(), context, state.mem); v.mkString(out.view(), context);
} }
static RegisterPrimOp primop_toJSON({ static RegisterPrimOp primop_toJSON({
@@ -2765,7 +2753,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId
/* Call the filter function. The first argument is the path, the /* Call the filter function. The first argument is the path, the
second is a string indicating the type of the file. */ second is a string indicating the type of the file. */
Value arg1; Value arg1;
arg1.mkString(path.path.abs(), mem); arg1.mkString(path.path.abs());
// assert that type is not "unknown" // assert that type is not "unknown"
Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))}; Value * args[]{&arg1, const_cast<Value *>(&fileTypeToString(*this, st.type))};
@@ -4415,7 +4403,7 @@ static void prim_toString(EvalState & state, const PosIdx pos, Value ** args, Va
NixStringContext context; NixStringContext context;
auto s = state.coerceToString( auto s = state.coerceToString(
pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false); pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false);
v.mkString(*s, context, state.mem); v.mkString(*s, context);
} }
static RegisterPrimOp primop_toString({ static RegisterPrimOp primop_toString({
@@ -4475,7 +4463,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
if (len == 0) { if (len == 0) {
state.forceValue(*args[2], pos); state.forceValue(*args[2], pos);
if (args[2]->type() == nString) { if (args[2]->type() == nString) {
v.mkStringNoCopy(""_sds, args[2]->context()); v.mkStringNoCopy("", args[2]->context());
return; return;
} }
} }
@@ -4488,7 +4476,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
auto s = state.coerceToString( auto s = state.coerceToString(
pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context, state.mem); v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context);
} }
static RegisterPrimOp primop_substring({ static RegisterPrimOp primop_substring({
@@ -4543,7 +4531,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value ** args,
auto s = auto s =
state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false), state.mem); v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false));
} }
static RegisterPrimOp primop_hashString({ static RegisterPrimOp primop_hashString({
@@ -4576,7 +4564,7 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value ** args,
HashFormat hf = parseHashFormat( HashFormat hf = parseHashFormat(
state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI), state.mem); v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI));
} }
static RegisterPrimOp primop_convertHash({ static RegisterPrimOp primop_convertHash({
@@ -4886,7 +4874,7 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value **
"while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"); "while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep");
} }
v.mkString(res, context, state.mem); v.mkString(res, context);
} }
static RegisterPrimOp primop_concatStringsSep({ static RegisterPrimOp primop_concatStringsSep({
@@ -4961,7 +4949,7 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** ar
} }
} }
v.mkString(res, context, state.mem); v.mkString(res, context);
} }
static RegisterPrimOp primop_replaceStrings({ static RegisterPrimOp primop_replaceStrings({
@@ -4994,8 +4982,8 @@ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value ** args
state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName"); state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName");
DrvName parsed(name); DrvName parsed(name);
auto attrs = state.buildBindings(2); auto attrs = state.buildBindings(2);
attrs.alloc(state.s.name).mkString(parsed.name, state.mem); attrs.alloc(state.s.name).mkString(parsed.name);
attrs.alloc("version").mkString(parsed.version, state.mem); attrs.alloc("version").mkString(parsed.version);
v.mkAttrs(attrs); v.mkAttrs(attrs);
} }
@@ -5050,7 +5038,7 @@ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value ** args
} }
auto list = state.buildList(components.size()); auto list = state.buildList(components.size());
for (const auto & [n, component] : enumerate(components)) for (const auto & [n, component] : enumerate(components))
(list[n] = state.allocValue())->mkString(std::move(component), state.mem); (list[n] = state.allocValue())->mkString(std::move(component));
v.mkList(list); v.mkList(list);
} }
@@ -5194,7 +5182,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
}); });
if (!settings.pureEval) if (!settings.pureEval)
v.mkString(settings.getCurrentSystem(), mem); v.mkString(settings.getCurrentSystem());
addConstant( addConstant(
"__currentSystem", "__currentSystem",
v, v,
@@ -5226,7 +5214,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
.impureOnly = true, .impureOnly = true,
}); });
v.mkString(nixVersion, mem); v.mkString(nixVersion);
addConstant( addConstant(
"__nixVersion", "__nixVersion",
v, v,
@@ -5251,7 +5239,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
)", )",
}); });
v.mkString(store->storeDir, mem); v.mkString(store->storeDir);
addConstant( addConstant(
"__storeDir", "__storeDir",
v, v,
@@ -5316,8 +5304,8 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings)
auto list = buildList(lookupPath.elements.size()); auto list = buildList(lookupPath.elements.size());
for (const auto & [n, i] : enumerate(lookupPath.elements)) { for (const auto & [n, i] : enumerate(lookupPath.elements)) {
auto attrs = buildBindings(2); auto attrs = buildBindings(2);
attrs.alloc("path").mkString(i.path.s, mem); attrs.alloc("path").mkString(i.path.s);
attrs.alloc("prefix").mkString(i.prefix.s, mem); attrs.alloc("prefix").mkString(i.prefix.s);
(list[n] = allocValue())->mkAttrs(attrs); (list[n] = allocValue())->mkAttrs(attrs);
} }
v.mkList(list); v.mkList(list);
+5 -5
View File
@@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
NixStringContext context; NixStringContext context;
auto s = state.coerceToString( auto s = state.coerceToString(
pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext");
v.mkString(*s, state.mem); v.mkString(*s);
} }
static RegisterPrimOp primop_unsafeDiscardStringContext({ static RegisterPrimOp primop_unsafeDiscardStringContext({
@@ -69,7 +69,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
} }
} }
v.mkString(*s, context2, state.mem); v.mkString(*s, context2);
} }
static RegisterPrimOp primop_unsafeDiscardOutputDependency( static RegisterPrimOp primop_unsafeDiscardOutputDependency(
@@ -137,7 +137,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
context.begin()->raw)}), context.begin()->raw)}),
}; };
v.mkString(*s, context2, state.mem); v.mkString(*s, context2);
} }
static RegisterPrimOp primop_addDrvOutputDependencies( static RegisterPrimOp primop_addDrvOutputDependencies(
@@ -218,7 +218,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
if (!info.second.outputs.empty()) { if (!info.second.outputs.empty()) {
auto list = state.buildList(info.second.outputs.size()); auto list = state.buildList(info.second.outputs.size());
for (const auto & [i, output] : enumerate(info.second.outputs)) for (const auto & [i, output] : enumerate(info.second.outputs))
(list[i] = state.allocValue())->mkString(output, state.mem); (list[i] = state.allocValue())->mkString(output);
infoAttrs.alloc(state.s.outputs).mkList(list); infoAttrs.alloc(state.s.outputs).mkList(list);
} }
attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
@@ -321,7 +321,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg
} }
} }
v.mkString(orig, context, state.mem); v.mkString(orig, context);
} }
static RegisterPrimOp primop_appendContext({.name = "__appendContext", .arity = 2, .fun = prim_appendContext}); static RegisterPrimOp primop_appendContext({.name = "__appendContext", .arity = 2, .fun = prim_appendContext});
+4 -4
View File
@@ -81,17 +81,17 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
attrs.insert_or_assign("rev", rev->gitRev()); attrs.insert_or_assign("rev", rev->gitRev());
auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store); auto [storePath, input2] = input.fetchToStore(state.store);
auto attrs2 = state.buildBindings(8); auto attrs2 = state.buildBindings(8);
state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath)); state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));
if (input2.getRef()) if (input2.getRef())
attrs2.alloc("branch").mkString(*input2.getRef(), state.mem); attrs2.alloc("branch").mkString(*input2.getRef());
// Backward compatibility: set 'rev' to // Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree. // 0000000000000000000000000000000000000000 for a dirty tree.
auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1)); auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1));
attrs2.alloc("rev").mkString(rev2.gitRev(), state.mem); attrs2.alloc("rev").mkString(rev2.gitRev());
attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12), state.mem); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12));
if (auto revCount = input2.getRevCount()) if (auto revCount = input2.getRevCount())
attrs2.alloc("revCount").mkInt(*revCount); attrs2.alloc("revCount").mkInt(*revCount);
v.mkAttrs(attrs2); v.mkAttrs(attrs2);
+215 -113
View File
@@ -35,7 +35,7 @@ void emitTreeAttrs(
// FIXME: support arbitrary input attributes. // FIXME: support arbitrary input attributes.
if (auto narHash = input.getNarHash()) if (auto narHash = input.getNarHash())
attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true), state.mem); attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (input.getType() == "git") if (input.getType() == "git")
attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); attrs.alloc("submodules").mkBool(fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
@@ -43,13 +43,13 @@ void emitTreeAttrs(
if (!forceDirty) { if (!forceDirty) {
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
attrs.alloc("rev").mkString(rev->gitRev(), state.mem); attrs.alloc("rev").mkString(rev->gitRev());
attrs.alloc("shortRev").mkString(rev->gitShortRev(), state.mem); attrs.alloc("shortRev").mkString(rev->gitShortRev());
} else if (emptyRevFallback) { } else if (emptyRevFallback) {
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(HashAlgorithm::SHA1); auto emptyHash = Hash(HashAlgorithm::SHA1);
attrs.alloc("rev").mkString(emptyHash.gitRev(), state.mem); attrs.alloc("rev").mkString(emptyHash.gitRev());
attrs.alloc("shortRev").mkString(emptyHash.gitShortRev(), state.mem); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev());
} }
if (auto revCount = input.getRevCount()) if (auto revCount = input.getRevCount())
@@ -59,14 +59,13 @@ void emitTreeAttrs(
} }
if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) { if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) {
attrs.alloc("dirtyRev").mkString(*dirtyRev, state.mem); attrs.alloc("dirtyRev").mkString(*dirtyRev);
attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"), state.mem); attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev"));
} }
if (auto lastModified = input.getLastModified()) { if (auto lastModified = input.getLastModified()) {
attrs.alloc("lastModified").mkInt(*lastModified); attrs.alloc("lastModified").mkInt(*lastModified);
attrs.alloc("lastModifiedDate") attrs.alloc("lastModifiedDate").mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
.mkString(fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")), state.mem);
} }
v.mkAttrs(attrs); v.mkAttrs(attrs);
@@ -83,7 +82,7 @@ struct FetchTreeParams
static void fetchTree( static void fetchTree(
EvalState & state, const PosIdx pos, Value ** args, Value & v, const FetchTreeParams & params = FetchTreeParams{}) EvalState & state, const PosIdx pos, Value ** args, Value & v, const FetchTreeParams & params = FetchTreeParams{})
{ {
fetchers::Input input{}; fetchers::Input input{state.fetchSettings};
NixStringContext context; NixStringContext context;
std::optional<std::string> type; std::optional<std::string> type;
auto fetcher = params.isFetchGit ? "fetchGit" : "fetchTree"; auto fetcher = params.isFetchGit ? "fetchGit" : "fetchTree";
@@ -151,6 +150,11 @@ static void fetchTree(
attrs.emplace("exportIgnore", Explicit<bool>{true}); attrs.emplace("exportIgnore", Explicit<bool>{true});
} }
// fetchTree should fetch git repos with shallow = true by default
if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) {
attrs.emplace("shallow", Explicit<bool>{true});
}
if (!params.allowNameArgument) if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher) state.error<EvalError>("argument 'name' isn’t supported in call to '%s'", fetcher)
@@ -190,9 +194,9 @@ static void fetchTree(
} }
if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first; input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first;
if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) { if (state.settings.pureEval && !input.isLocked()) {
if (input.getNarHash()) if (input.getNarHash())
warn( warn(
"Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " "Input '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
@@ -215,8 +219,7 @@ static void fetchTree(
throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string());
} }
auto cachedInput = auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No);
state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No);
auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor); auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
@@ -231,127 +234,229 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
static RegisterPrimOp primop_fetchTree({ static RegisterPrimOp primop_fetchTree({
.name = "fetchTree", .name = "fetchTree",
.args = {"input"}, .args = {"input"},
.doc = []() -> std::string { .doc = R"(
std::string doc = stripIndentation(R"( Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
- the resulting fixed-output [store path](@docroot@/store/store-path.md) - the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes --> - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
*input* must be an attribute set with the following attributes: *input* must be an attribute set with the following attributes:
- `type` (String, required) - `type` (String, required)
One of the [supported source types](#source-types). One of the [supported source types](#source-types).
This determines other required and allowed input attributes. This determines other required and allowed input attributes.
- `narHash` (String, optional) - `narHash` (String, optional)
The `narHash` parameter can be used to substitute the source of the tree. The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent. That is, `fetchTree` is idempotent.
Downloads are cached in `$XDG_CACHE_HOME/nix`. Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source is fetched from the network if both are true: The remote source is fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
> **Note** > **Note**
>
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
## Source types
The following source types and associated input attributes are supported.
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
- `"file"`
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
> >
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. > ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) - `http`
## Source types Insecure HTTP transfer for legacy sources.
The following source types and associated input attributes are supported. > **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
<!-- TODO: It would be soooo much more predictable to work with (and - `file`
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
)");
auto indentString = [](std::string const & str, std::string const & indent) { A file on the local file system.
std::string result;
std::istringstream stream(str);
std::string line;
bool first = true;
while (std::getline(stream, line)) {
if (!first)
result += "\n";
result += indent + line;
first = false;
}
return result;
};
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) { > **Example**
doc += "\n- `" + quoteString(schemeName, '"') + "`\n\n"; >
doc += indentString(scheme->schemeDescription(), " "); > ```nix
if (!doc.empty() && doc.back() != '\n') > fetchTree {
doc += "\n"; > type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
for (const auto & [attrName, attribute] : scheme->allowedAttrs()) { - `"tarball"`
doc += "\n - `" + attrName + "` (" + attribute.type + ", "
+ (attribute.required ? "required" : "optional") + ")\n\n";
doc += indentString(stripIndentation(attribute.doc), " ");
if (!doc.empty() && doc.back() != '\n')
doc += "\n";
}
}
doc += "\n" + stripIndentation(R"( Download a tar archive and extract it into the Nix store.
The following input types are still subject to change: This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `"path"` - `url` (String, required)
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). > **Example**
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. >
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example** > **Example**
> >
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix > ```nix
> builtins.fetchTree { > fetchTree {
> type = "github"; > type = "git";
> owner = "NixOS"; > url = "git@github.com:NixOS/nixpkgs.git";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> } > }
> ``` > ```
> **Example** > **Note**
> >
> Fetch the same GitHub repository using the URL-like syntax: > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)");
return doc; - `ref` (String, optional)
}(),
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `lfs` (Bool, optional)
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
- `allRefs` (Bool, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
The following input types are still subject to change:
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
> **Example**
>
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
.fun = prim_fetchTree, .fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree, .experimentalFeature = Xp::FetchTree,
}); });
@@ -362,13 +467,10 @@ void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value ** args, Val
} }
static RegisterPrimOp primop_fetchFinalTree({ static RegisterPrimOp primop_fetchFinalTree({
.name = "__fetchFinalTree", .name = "fetchFinalTree",
.args = {"input"}, .args = {"input"},
.doc = R"(
Like `fetchTree`, but does not return any additional fetcher attributes (like `revCount`).
This allows inputs to be substituted if `narHash` is specified.
)",
.fun = prim_fetchFinalTree, .fun = prim_fetchFinalTree,
.internal = true,
}); });
static void fetch( static void fetch(
@@ -479,10 +581,10 @@ static void fetch(
auto storePath = unpack ? fetchToStore( auto storePath = unpack ? fetchToStore(
state.fetchSettings, state.fetchSettings,
*state.store, *state.store,
fetchers::downloadTarball(*state.store, state.fetchSettings, *url), fetchers::downloadTarball(state.store, state.fetchSettings, *url),
FetchMode::Copy, FetchMode::Copy,
name) name)
: fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath; : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
if (expectedHash) { if (expectedHash) {
auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash
+3 -4
View File
@@ -1,6 +1,5 @@
#include "nix/expr/primops.hh" #include "nix/expr/primops.hh"
#include "nix/expr/eval-inline.hh" #include "nix/expr/eval-inline.hh"
#include "nix/expr/static-string-data.hh"
#include "expr-config-private.hh" #include "expr-config-private.hh"
@@ -126,7 +125,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
case toml::value_t::string: { case toml::value_t::string: {
auto s = toml::get<std::string_view>(t); auto s = toml::get<std::string_view>(t);
forceNoNullByte(s); forceNoNullByte(s);
v.mkString(s, state.mem); v.mkString(s);
} break; } break;
case toml::value_t::local_datetime: case toml::value_t::local_datetime:
case toml::value_t::offset_datetime: case toml::value_t::offset_datetime:
@@ -137,12 +136,12 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
normalizeDatetimeFormat(t); normalizeDatetimeFormat(t);
#endif #endif
auto attrs = state.buildBindings(2); auto attrs = state.buildBindings(2);
attrs.alloc("_type").mkStringNoCopy("timestamp"_sds); attrs.alloc("_type").mkStringNoCopy("timestamp");
std::ostringstream s; std::ostringstream s;
s << t; s << t;
auto str = s.view(); auto str = s.view();
forceNoNullByte(str); forceNoNullByte(str);
attrs.alloc("value").mkString(str, state.mem); attrs.alloc("value").mkString(str);
v.mkAttrs(attrs); v.mkAttrs(attrs);
} else { } else {
throw std::runtime_error("Dates and times are not supported"); throw std::runtime_error("Dates and times are not supported");
-2
View File
@@ -16,8 +16,6 @@ json printValueAsJSON(
{ {
checkInterrupt(); checkInterrupt();
auto _level = state.addCallDepth(pos);
if (strict) if (strict)
state.forceValue(v, pos); state.forceValue(v, pos);
+9 -9
View File
@@ -1,19 +1,19 @@
#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-utils.hh"
#include "nix/util/file-system.hh" #include "nix/util/file-system.hh"
#include <gmock/gmock.h> #include <gmock/gmock.h>
#include <git2/global.h> #include <git2-experimental/global.h>
#include <git2/repository.h> #include <git2-experimental/repository.h>
#include <git2/signature.h> #include <git2-experimental/signature.h>
#include <git2/types.h> #include <git2-experimental/types.h>
#include <git2/object.h> #include <git2-experimental/object.h>
#include <git2/tag.h> #include <git2-experimental/tag.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include "nix/util/fs-sink.hh" #include "nix/util/fs-sink.hh"
#include "nix/util/serialise.hh" #include "nix/util/serialise.hh"
#include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/git-lfs-fetch.hh"
#include <git2/blob.h> #include <git2-experimental/blob.h>
#include <git2/tree.h> #include <git2-experimental/tree.h>
namespace nix { namespace nix {
@@ -92,7 +92,7 @@ TEST_F(GitUtilsTest, sink_basic)
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
auto result = repo->dereferenceSingletonDirectory(sink->flush()); auto result = repo->dereferenceSingletonDirectory(sink->flush());
auto accessor = repo->getAccessor(result, {}, getRepoName()); auto accessor = repo->getAccessor(result, false, getRepoName());
auto entries = accessor->readDirectory(CanonPath::root); auto entries = accessor->readDirectory(CanonPath::root);
ASSERT_EQ(entries.size(), 5u); ASSERT_EQ(entries.size(), 5u);
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
+2 -2
View File
@@ -5,7 +5,7 @@
#include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-utils.hh"
#include <git2.h> #include <git2-experimental.h>
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <filesystem> #include <filesystem>
@@ -196,7 +196,7 @@ TEST_F(GitTest, submodulePeriodSupport)
{"ref", "main"}, {"ref", "main"},
}); });
auto [accessor, i] = input.getAccessor(settings, *store); auto [accessor, i] = input.getAccessor(store);
ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n"); ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n");
} }
-61
View File
@@ -1,61 +0,0 @@
#include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/attrs.hh"
#include "nix/fetchers/fetchers.hh"
#include <gtest/gtest.h>
#include <string>
namespace nix {
using fetchers::Attr;
struct InputFromAttrsTestCase
{
fetchers::Attrs attrs;
std::string expectedUrl;
std::string description;
fetchers::Attrs expectedAttrs = attrs;
};
class InputFromAttrsTest : public ::testing::WithParamInterface<InputFromAttrsTestCase>, public ::testing::Test
{};
TEST_P(InputFromAttrsTest, attrsAreCorrectAndRoundTrips)
{
fetchers::Settings fetchSettings;
const auto & testCase = GetParam();
auto input = fetchers::Input::fromAttrs(fetchSettings, fetchers::Attrs(testCase.attrs));
EXPECT_EQ(input.toAttrs(), testCase.expectedAttrs);
EXPECT_EQ(input.toURLString(), testCase.expectedUrl);
auto input2 = fetchers::Input::fromAttrs(fetchSettings, input.toAttrs());
EXPECT_EQ(input, input2);
EXPECT_EQ(input.toAttrs(), input2.toAttrs());
}
INSTANTIATE_TEST_SUITE_P(
InputFromAttrs,
InputFromAttrsTest,
::testing::Values(
// Test for issue #14429.
InputFromAttrsTestCase{
.attrs =
{
{"url", Attr("git+ssh://git@github.com/NixOS/nixpkgs")},
{"type", Attr("git")},
},
.expectedUrl = "git+ssh://git@github.com/NixOS/nixpkgs",
.description = "strips_git_plus_prefix",
.expectedAttrs =
{
{"url", Attr("ssh://git@github.com/NixOS/nixpkgs")},
{"type", Attr("git")},
},
}),
[](const ::testing::TestParamInfo<InputFromAttrsTestCase> & info) { return info.param.description; });
} // namespace nix
+1 -2
View File
@@ -33,7 +33,7 @@ deps_private += rapidcheck
gtest = dependency('gtest', main : true) gtest = dependency('gtest', main : true)
deps_private += gtest deps_private += gtest
libgit2 = dependency('libgit2') libgit2 = dependency('libgit2-experimental')
deps_private += libgit2 deps_private += libgit2
subdir('nix-meson-build-support/common') subdir('nix-meson-build-support/common')
@@ -42,7 +42,6 @@ sources = files(
'access-tokens.cc', 'access-tokens.cc',
'git-utils.cc', 'git-utils.cc',
'git.cc', 'git.cc',
'input.cc',
'nix_api_fetchers.cc', 'nix_api_fetchers.cc',
'public-key.cc', 'public-key.cc',
) )
+2 -1
View File
@@ -1,5 +1,6 @@
#include "nix/fetchers/attrs.hh" #include "nix/fetchers/attrs.hh"
#include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetchers.hh"
#include "nix/fetchers/git-utils.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -111,7 +112,7 @@ StringMap attrsToQuery(const Attrs & attrs)
Hash getRevAttr(const Attrs & attrs, const std::string & name) Hash getRevAttr(const Attrs & attrs, const std::string & name)
{ {
return Hash::parseAny(getStrAttr(attrs, name), HashAlgorithm::SHA1); return parseGitHash(getStrAttr(attrs, name));
} }
} // namespace nix::fetchers } // namespace nix::fetchers
+49 -34
View File
@@ -6,7 +6,6 @@
#include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetch-settings.hh"
#include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetch-to-store.hh"
#include "nix/util/url.hh" #include "nix/util/url.hh"
#include "nix/util/archive.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -27,9 +26,18 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
throw Error("Input scheme with name %s already registered", schemeName); throw Error("Input scheme with name %s already registered", schemeName);
} }
const InputSchemeMap & getAllInputSchemes() nlohmann::json dumpRegisterInputSchemeInfo()
{ {
return inputSchemes(); using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : inputSchemes()) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
} }
Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree) Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)
@@ -81,7 +89,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
// but not all of them. Doing this is to support those other // but not all of them. Doing this is to support those other
// operations which are supposed to be robust on // operations which are supposed to be robust on
// unknown/uninterpretable inputs. // unknown/uninterpretable inputs.
Input input; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
fixupInput(input); fixupInput(input);
return input; return input;
@@ -111,7 +119,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
return std::move(*res); return std::move(*res);
} }
std::optional<std::string> Input::getFingerprint(Store & store) const std::optional<std::string> Input::getFingerprint(ref<Store> store) const
{ {
if (!scheme) if (!scheme)
return std::nullopt; return std::nullopt;
@@ -151,9 +159,9 @@ bool Input::isDirect() const
return !scheme || scheme->isDirect(*this); return !scheme || scheme->isDirect(*this);
} }
bool Input::isLocked(const Settings & settings) const bool Input::isLocked() const
{ {
return scheme && scheme->isLocked(settings, *this); return scheme && scheme->isLocked(*this);
} }
bool Input::isFinal() const bool Input::isFinal() const
@@ -190,19 +198,19 @@ bool Input::contains(const Input & other) const
} }
// FIXME: remove // FIXME: remove
std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store & store) const std::pair<StorePath, Input> Input::fetchToStore(ref<Store> store) const
{ {
if (!scheme) if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
auto [storePath, input] = [&]() -> std::pair<StorePath, Input> { auto [storePath, input] = [&]() -> std::pair<StorePath, Input> {
try { try {
auto [accessor, result] = getAccessorUnchecked(settings, store); auto [accessor, result] = getAccessorUnchecked(store);
auto storePath = auto storePath =
nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName()); nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
auto narHash = store.queryPathInfo(storePath)->narHash; auto narHash = store->queryPathInfo(storePath)->narHash;
result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
result.attrs.insert_or_assign("__final", Explicit<bool>(true)); result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -269,16 +277,30 @@ void Input::checkLocks(Input specified, Input & result)
} }
} }
if (auto prevLastModified = specified.getLastModified()) {
if (result.getLastModified() != prevLastModified)
throw Error(
"'lastModified' attribute mismatch in input '%s', expected %d, got %d",
result.to_string(),
*prevLastModified,
result.getLastModified().value_or(-1));
}
if (auto prevRev = specified.getRev()) { if (auto prevRev = specified.getRev()) {
if (result.getRev() != prevRev) if (result.getRev() != prevRev)
throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev()); throw Error("'rev' attribute mismatch in input '%s', expected %s", result.to_string(), prevRev->gitRev());
} }
if (auto prevRevCount = specified.getRevCount()) {
if (result.getRevCount() != prevRevCount)
throw Error("'revCount' attribute mismatch in input '%s', expected %d", result.to_string(), *prevRevCount);
}
} }
std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const std::pair<ref<SourceAccessor>, Input> Input::getAccessor(ref<Store> store) const
{ {
try { try {
auto [accessor, result] = getAccessorUnchecked(settings, store); auto [accessor, result] = getAccessorUnchecked(store);
result.attrs.insert_or_assign("__final", Explicit<bool>(true)); result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -291,7 +313,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settin
} }
} }
std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, Store & store) const std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(ref<Store> store) const
{ {
// FIXME: cache the accessor // FIXME: cache the accessor
@@ -311,13 +333,13 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
*/ */
if (isFinal() && getNarHash()) { if (isFinal() && getNarHash()) {
try { try {
auto storePath = computeStorePath(store); auto storePath = computeStorePath(*store);
store.ensurePath(storePath); store->ensurePath(storePath);
debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath)); debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
auto accessor = store.requireStoreObjectAccessor(storePath); auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->fingerprint = getFingerprint(store); accessor->fingerprint = getFingerprint(store);
@@ -327,7 +349,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
if (accessor->fingerprint) { if (accessor->fingerprint) {
ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/"); auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
settings.getCache()->upsert(cacheKey, store, {}, storePath); settings->getCache()->upsert(cacheKey, *store, {}, storePath);
} }
accessor->setPathDisplay("«" + to_string() + "»"); accessor->setPathDisplay("«" + to_string() + "»");
@@ -338,7 +360,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
} }
} }
auto [accessor, result] = scheme->getAccessor(settings, store, *this); auto [accessor, result] = scheme->getAccessor(store, *this);
if (!accessor->fingerprint) if (!accessor->fingerprint)
accessor->fingerprint = result.getFingerprint(store); accessor->fingerprint = result.getFingerprint(store);
@@ -355,10 +377,10 @@ Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash>
return scheme->applyOverrides(*this, ref, rev); return scheme->applyOverrides(*this, ref, rev);
} }
void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const void Input::clone(const Path & destDir) const
{ {
assert(scheme); assert(scheme);
scheme->clone(settings, store, *this, destDir); scheme->clone(*this, destDir);
} }
std::optional<std::filesystem::path> Input::getSourcePath() const std::optional<std::filesystem::path> Input::getSourcePath() const
@@ -425,6 +447,9 @@ std::optional<Hash> Input::getRev() const
} catch (BadHash & e) { } catch (BadHash & e) {
// Default to sha1 for backwards compatibility with existing // Default to sha1 for backwards compatibility with existing
// usages (e.g. `builtins.fetchTree` calls or flake inputs). // usages (e.g. `builtins.fetchTree` calls or flake inputs).
//
// Note that means that for SHA-256 git repos, prefixing
// must be used.
hash = Hash::parseAny(*s, HashAlgorithm::SHA1); hash = Hash::parseAny(*s, HashAlgorithm::SHA1);
} }
} }
@@ -471,19 +496,9 @@ void InputScheme::putFile(
throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
} }
void InputScheme::clone( void InputScheme::clone(const Input & input, const Path & destDir) const
const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
{ {
if (std::filesystem::exists(destDir)) throw Error("do not know how to clone input '%s'", input.to_string());
throw Error("cannot clone into existing path %s", destDir);
auto [accessor, input2] = getAccessor(settings, store, input);
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
RestoreSink sink(/*startFsync=*/false);
sink.dstPath = destDir;
copyRecursive(*accessor, CanonPath::root, sink, CanonPath::root);
} }
std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
+5 -5
View File
@@ -7,10 +7,10 @@
#include "nix/util/hash.hh" #include "nix/util/hash.hh"
#include "nix/store/ssh.hh" #include "nix/store/ssh.hh"
#include <git2/attr.h> #include <git2-experimental/attr.h>
#include <git2/config.h> #include <git2-experimental/config.h>
#include <git2/errors.h> #include <git2-experimental/errors.h>
#include <git2/remote.h> #include <git2-experimental/remote.h>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
auto url = api.endpoint + "/objects/batch"; auto url = api.endpoint + "/objects/batch";
const auto & authHeader = api.authHeader; const auto & authHeader = api.authHeader;
FileTransferRequest request(parseURL(url)); FileTransferRequest request(parseURL(url));
request.method = HttpMethod::Post; request.method = HttpMethod::POST;
Headers headers; Headers headers;
if (authHeader.has_value()) if (authHeader.has_value())
headers.push_back({"Authorization", *authHeader}); headers.push_back({"Authorization", *authHeader});
+92 -54
View File
@@ -13,27 +13,27 @@
#include "nix/util/thread-pool.hh" #include "nix/util/thread-pool.hh"
#include "nix/util/pool.hh" #include "nix/util/pool.hh"
#include <git2/attr.h> #include <git2-experimental/attr.h>
#include <git2/blob.h> #include <git2-experimental/blob.h>
#include <git2/branch.h> #include <git2-experimental/branch.h>
#include <git2/commit.h> #include <git2-experimental/commit.h>
#include <git2/config.h> #include <git2-experimental/config.h>
#include <git2/describe.h> #include <git2-experimental/describe.h>
#include <git2/errors.h> #include <git2-experimental/errors.h>
#include <git2/global.h> #include <git2-experimental/global.h>
#include <git2/indexer.h> #include <git2-experimental/indexer.h>
#include <git2/object.h> #include <git2-experimental/object.h>
#include <git2/odb.h> #include <git2-experimental/odb.h>
#include <git2/refs.h> #include <git2-experimental/refs.h>
#include <git2/remote.h> #include <git2-experimental/remote.h>
#include <git2/repository.h> #include <git2-experimental/repository.h>
#include <git2/revparse.h> #include <git2-experimental/revparse.h>
#include <git2/status.h> #include <git2-experimental/status.h>
#include <git2/submodule.h> #include <git2-experimental/submodule.h>
#include <git2/sys/odb_backend.h> #include <git2-experimental/sys/odb_backend.h>
#include <git2/sys/mempack.h> #include <git2-experimental/sys/mempack.h>
#include <git2/tag.h> #include <git2-experimental/tag.h>
#include <git2/tree.h> #include <git2-experimental/tree.h>
#include <boost/unordered/concurrent_flat_set.hpp> #include <boost/unordered/concurrent_flat_set.hpp>
#include <boost/unordered/unordered_flat_map.hpp> #include <boost/unordered/unordered_flat_map.hpp>
@@ -91,10 +91,21 @@ typedef std::unique_ptr<git_indexer, Deleter<git_indexer_free>> Indexer;
Hash toHash(const git_oid & oid) Hash toHash(const git_oid & oid)
{ {
#ifdef GIT_EXPERIMENTAL_SHA256 HashAlgorithm algo;
assert(oid.type == GIT_OID_SHA1); #pragma GCC diagnostic push
#endif #pragma GCC diagnostic ignored "-Wswitch-enum"
Hash hash(HashAlgorithm::SHA1); switch (oid.type) {
case GIT_OID_SHA1:
algo = HashAlgorithm::SHA1;
break;
case GIT_OID_SHA256:
algo = HashAlgorithm::SHA256;
break;
default:
unreachable();
}
#pragma GCC diagnostic pop
Hash hash(algo);
memcpy(hash.hash, oid.id, hash.hashSize); memcpy(hash.hash, oid.id, hash.hashSize);
return hash; return hash;
} }
@@ -111,7 +122,21 @@ static void initLibGit2()
git_oid hashToOID(const Hash & hash) git_oid hashToOID(const Hash & hash)
{ {
git_oid oid; git_oid oid;
if (git_oid_fromstr(&oid, hash.gitRev().c_str())) git_oid_t t;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch-enum"
switch (hash.algo) {
case HashAlgorithm::SHA1:
t = GIT_OID_SHA1;
break;
case HashAlgorithm::SHA256:
t = GIT_OID_SHA256;
break;
default:
throw Error("unsupported hash algorithm for Git: %s", printHashAlgo(hash.algo));
}
#pragma GCC diagnostic pop
if (git_oid_fromstr(&oid, hash.gitRev().c_str(), t))
throw Error("cannot convert '%s' to a Git OID", hash.gitRev()); throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
return oid; return oid;
} }
@@ -304,7 +329,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
// (synchronously on the git_packbuilder_write_buf thread) // (synchronously on the git_packbuilder_write_buf thread)
Indexer indexer; Indexer indexer;
git_indexer_progress stats; git_indexer_progress stats;
if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) git_indexer_options indexer_opts = GIT_INDEXER_OPTIONS_INIT;
if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), &indexer_opts))
throw Error("creating git packfile indexer: %s", git_error_last()->message); throw Error("creating git packfile indexer: %s", git_error_last()->message);
// TODO: provide index callback for checkInterrupt() termination // TODO: provide index callback for checkInterrupt() termination
@@ -541,15 +567,14 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
} }
/** /**
* A 'GitSourceAccessor' with no regard for export-ignore. * A 'GitSourceAccessor' with no regard for export-ignore or any other transformations.
*/ */
ref<GitSourceAccessor> getRawAccessor(const Hash & rev, const GitAccessorOptions & options); ref<GitSourceAccessor> getRawAccessor(const Hash & rev, bool smudgeLfs = false);
ref<SourceAccessor> ref<SourceAccessor>
getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) override; getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) override;
ref<SourceAccessor> ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
getAccessor(const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError e) override;
ref<GitFileSystemObjectSink> getFileSystemObjectSink() override; ref<GitFileSystemObjectSink> getFileSystemObjectSink() override;
@@ -669,7 +694,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override
{ {
auto accessor = getAccessor(treeHash, {}, ""); auto accessor = getAccessor(treeHash, false, "");
fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
@@ -717,17 +742,15 @@ struct GitSourceAccessor : SourceAccessor
ref<GitRepoImpl> repo; ref<GitRepoImpl> repo;
Object root; Object root;
std::optional<lfs::Fetch> lfsFetch = std::nullopt; std::optional<lfs::Fetch> lfsFetch = std::nullopt;
GitAccessorOptions options;
}; };
Sync<State> state_; Sync<State> state_;
GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, const GitAccessorOptions & options) GitSourceAccessor(ref<GitRepoImpl> repo_, const Hash & rev, bool smudgeLfs)
: state_{State{ : state_{State{
.repo = repo_, .repo = repo_,
.root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()), .root = peelToTreeOrBlob(lookupObject(*repo_, hashToOID(rev)).get()),
.lfsFetch = options.smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt, .lfsFetch = smudgeLfs ? std::make_optional(lfs::Fetch(*repo_, hashToOID(rev))) : std::nullopt,
.options = options,
}} }}
{ {
} }
@@ -1257,26 +1280,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
} }
}; };
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options) ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, bool smudgeLfs)
{ {
auto self = ref<GitRepoImpl>(shared_from_this()); auto self = ref<GitRepoImpl>(shared_from_this());
return make_ref<GitSourceAccessor>(self, rev, options); return make_ref<GitSourceAccessor>(self, rev, smudgeLfs);
} }
ref<SourceAccessor> ref<SourceAccessor>
GitRepoImpl::getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs)
{ {
auto self = ref<GitRepoImpl>(shared_from_this()); auto self = ref<GitRepoImpl>(shared_from_this());
ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, options); ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev, smudgeLfs);
rawGitAccessor->setPathDisplay(std::move(displayPrefix)); rawGitAccessor->setPathDisplay(std::move(displayPrefix));
if (options.exportIgnore) if (exportIgnore)
return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev); return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
else else
return rawGitAccessor; return rawGitAccessor;
} }
ref<SourceAccessor> GitRepoImpl::getAccessor( ref<SourceAccessor>
const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
{ {
auto self = ref<GitRepoImpl>(shared_from_this()); auto self = ref<GitRepoImpl>(shared_from_this());
ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create( ref<SourceAccessor> fileAccessor = AllowListSourceAccessor::create(
@@ -1286,9 +1309,10 @@ ref<SourceAccessor> GitRepoImpl::getAccessor(
boost::unordered_flat_set<CanonPath>{CanonPath::root}, boost::unordered_flat_set<CanonPath>{CanonPath::root},
std::move(makeNotAllowedError)) std::move(makeNotAllowedError))
.cast<SourceAccessor>(); .cast<SourceAccessor>();
if (options.exportIgnore) if (exportIgnore)
fileAccessor = make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt); return make_ref<GitExportIgnoreSourceAccessor>(self, fileAccessor, std::nullopt);
return fileAccessor; else
return fileAccessor;
} }
ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink() ref<GitFileSystemObjectSink> GitRepoImpl::getFileSystemObjectSink()
@@ -1301,7 +1325,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
/* Read the .gitmodules files from this revision. */ /* Read the .gitmodules files from this revision. */
CanonPath modulesFile(".gitmodules"); CanonPath modulesFile(".gitmodules");
auto accessor = getAccessor(rev, {.exportIgnore = exportIgnore}, ""); auto accessor = getAccessor(rev, exportIgnore, "");
if (!accessor->pathExists(modulesFile)) if (!accessor->pathExists(modulesFile))
return {}; return {};
@@ -1318,7 +1342,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
std::vector<std::tuple<Submodule, Hash>> result; std::vector<std::tuple<Submodule, Hash>> result;
auto rawAccessor = getRawAccessor(rev, {}); auto rawAccessor = getRawAccessor(rev);
for (auto & submodule : parseSubmodules(pathTemp)) { for (auto & submodule : parseSubmodules(pathTemp)) {
/* Filter out .gitmodules entries that don't exist or are not /* Filter out .gitmodules entries that don't exist or are not
@@ -1330,16 +1354,13 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
return result; return result;
} }
namespace fetchers { ref<GitRepo> getTarballCache()
ref<GitRepo> Settings::getTarballCache() const
{ {
static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
return GitRepo::openRepo(repoDir, true, true); return GitRepo::openRepo(repoDir, true, true);
} }
} // namespace fetchers
GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)
{ {
static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache; static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache;
@@ -1383,4 +1404,21 @@ bool isLegalRefName(const std::string & refName)
return false; return false;
} }
Hash parseGitHash(std::string_view hashStr)
{
HashAlgorithm algo;
switch (hashStr.size()) {
case 40:
algo = HashAlgorithm::SHA1;
break;
case 64:
algo = HashAlgorithm::SHA256;
break;
default:
throw Error(
"invalid git hash '%s': expected 40 (SHA1) or 64 (SHA256) hex characters, got %d", hashStr, hashStr.size());
}
return Hash::parseNonSRIUnprefixed(hashStr, algo);
}
} // namespace nix } // namespace nix
+56 -243
View File
@@ -168,6 +168,8 @@ struct GitInputScheme : InputScheme
return {}; return {};
auto url2(url); auto url2(url);
if (hasPrefix(url2.scheme, "git+"))
url2.scheme = std::string(url2.scheme, 4);
url2.query.clear(); url2.query.clear();
Attrs attrs; Attrs attrs;
@@ -194,183 +196,28 @@ struct GitInputScheme : InputScheme
return "git"; return "git";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
return stripIndentation(R"( return {
Fetch a Git tree and copy it to the Nix store. "url",
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). "ref",
)"); "rev",
} "shallow",
"submodules",
const std::map<std::string, AttributeInfo> & allowedAttrs() const override "lfs",
{ "exportIgnore",
static const std::map<std::string, AttributeInfo> attrs = { "lastModified",
{ "revCount",
"url", "narHash",
{ "allRefs",
.type = "String", "name",
.required = true, "dirtyRev",
.doc = R"( "dirtyShortRev",
The URL formats supported are the same as for Git itself. "verifyCommit",
"keytype",
> **Example** "publicKey",
> "publicKeys",
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but uses the *current file contents* of the Git working directory.
)",
},
},
{
"ref",
{
.type = "String",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
)",
},
},
{
"rev",
{
.type = "String",
.required = false,
.doc = R"(
A Git revision; a commit hash.
Default: the tip of `ref`
)",
},
},
{
"shallow",
{
.type = "Bool",
.required = false,
.doc = R"(
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
)",
},
},
{
"submodules",
{
.type = "Bool",
.required = false,
.doc = R"(
Also fetch submodules if available.
Default: `false`
)",
},
},
{
"lfs",
{
.type = "Bool",
.required = false,
.doc = R"(
Fetch any [Git LFS](https://git-lfs.com/) files.
Default: `false`
)",
},
},
{
"exportIgnore",
{},
},
{
"lastModified",
{
.type = "Integer",
.required = false,
.doc = R"(
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"revCount",
{
.type = "Integer",
.required = false,
.doc = R"(
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"narHash",
{},
},
{
"allRefs",
{
.type = "Bool",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
)",
},
},
{
"name",
{},
},
{
"dirtyRev",
{},
},
{
"dirtyShortRev",
{},
},
{
"verifyCommit",
{},
},
{
"keytype",
{},
},
{
"publicKey",
{},
},
{
"publicKeys",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -384,7 +231,7 @@ struct GitInputScheme : InputScheme
if (auto ref = maybeGetStrAttr(attrs, "ref"); ref && !isLegalRefName(*ref)) if (auto ref = maybeGetStrAttr(attrs, "ref"); ref && !isLegalRefName(*ref))
throw BadURL("invalid Git branch/tag name '%s'", *ref); throw BadURL("invalid Git branch/tag name '%s'", *ref);
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string(); input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string();
getShallowAttr(input); getShallowAttr(input);
@@ -433,8 +280,7 @@ struct GitInputScheme : InputScheme
return res; return res;
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto repoInfo = getRepoInfo(input); auto repoInfo = getRepoInfo(input);
@@ -778,25 +624,15 @@ struct GitInputScheme : InputScheme
} }
} }
/**
* Decide whether we can do a shallow clone, which is faster. This is possible if the user explicitly specified
* `shallow = true`, or if we already have a `revCount`.
*/
bool canDoShallow(const Input & input) const
{
bool shallow = getShallowAttr(input);
return shallow || input.getRevCount().has_value();
}
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const getAccessorFromCommit(ref<Store> store, RepoInfo & repoInfo, Input && input) const
{ {
assert(!repoInfo.workdirInfo.isDirty); assert(!repoInfo.workdirInfo.isDirty);
auto origRev = input.getRev(); auto origRev = input.getRev();
auto originalRef = input.getRef(); auto originalRef = input.getRef();
bool shallow = canDoShallow(input); bool shallow = getShallowAttr(input);
auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo, shallow);
input.attrs.insert_or_assign("ref", ref); input.attrs.insert_or_assign("ref", ref);
@@ -807,27 +643,11 @@ struct GitInputScheme : InputScheme
if (!input.getRev()) if (!input.getRev())
input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev());
} else { } else {
auto rev = input.getRev();
auto repoUrl = std::get<ParsedURL>(repoInfo.location); auto repoUrl = std::get<ParsedURL>(repoInfo.location);
std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow); std::filesystem::path cacheDir = getCachePath(repoUrl.to_string(), shallow);
repoDir = cacheDir; repoDir = cacheDir;
repoInfo.gitDir = "."; repoInfo.gitDir = ".";
/* If shallow = false, but we have a non-shallow repo that already contains the desired rev, then use that
* repo instead. */
std::filesystem::path cacheDirNonShallow = getCachePath(repoUrl.to_string(), false);
if (rev && shallow && pathExists(cacheDirNonShallow)) {
auto nonShallowRepo = GitRepo::openRepo(cacheDirNonShallow, true, true);
if (nonShallowRepo->hasObject(*rev)) {
debug(
"using non-shallow cached repo for '%s' since it contains rev '%s'",
repoUrl.to_string(),
rev->gitRev());
repoDir = cacheDirNonShallow;
goto have_rev;
}
}
std::filesystem::create_directories(cacheDir.parent_path()); std::filesystem::create_directories(cacheDir.parent_path());
PathLocks cacheDirLock({cacheDir.string()}); PathLocks cacheDirLock({cacheDir.string()});
@@ -843,7 +663,7 @@ struct GitInputScheme : InputScheme
/* If a rev was specified, we need to fetch if it's not in the /* If a rev was specified, we need to fetch if it's not in the
repo. */ repo. */
if (rev) { if (auto rev = input.getRev()) {
doFetch = !repo->hasObject(*rev); doFetch = !repo->hasObject(*rev);
} else { } else {
if (getAllRefsAttr(input)) { if (getAllRefsAttr(input)) {
@@ -857,6 +677,7 @@ struct GitInputScheme : InputScheme
} }
if (doFetch) { if (doFetch) {
bool shallow = getShallowAttr(input);
try { try {
auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*" auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*"
: input.getRev() ? input.getRev()->gitRev() : input.getRev() ? input.getRev()->gitRev()
@@ -884,7 +705,7 @@ struct GitInputScheme : InputScheme
warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg()); warn("could not update cached head '%s' for '%s'", ref, repoInfo.locationToArg());
} }
if (rev) { if (auto rev = input.getRev()) {
if (!repo->hasObject(*rev)) if (!repo->hasObject(*rev))
throw Error( throw Error(
"Cannot find Git revision '%s' in ref '%s' of repository '%s'! " "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
@@ -901,30 +722,23 @@ struct GitInputScheme : InputScheme
// the remainder // the remainder
} }
have_rev:
auto repo = GitRepo::openRepo(repoDir); auto repo = GitRepo::openRepo(repoDir);
auto isShallow = repo->isShallow();
if (isShallow && !getShallowAttr(input))
throw Error(
"'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
repoInfo.locationToArg());
// FIXME: check whether rev is an ancestor of ref? // FIXME: check whether rev is an ancestor of ref?
auto rev = *input.getRev(); auto rev = *input.getRev();
/* Skip lastModified computation if it's already supplied by the caller. input.attrs.insert_or_assign("lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev));
We don't care if they specify an incorrect value; it doesn't
matter for security, unlike narHash. */
if (!input.attrs.contains("lastModified"))
input.attrs.insert_or_assign("lastModified", getLastModified(settings, repoInfo, repoDir, rev));
/* Like lastModified, skip revCount if supplied by the caller. */ if (!getShallowAttr(input))
if (!shallow && !input.attrs.contains("revCount")) { input.attrs.insert_or_assign("revCount", getRevCount(*input.settings, repoInfo, repoDir, rev));
auto isShallow = repo->isShallow();
if (isShallow && !shallow)
throw Error(
"'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified",
repoInfo.locationToArg());
input.attrs.insert_or_assign("revCount", getRevCount(settings, repoInfo, repoDir, rev));
}
printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg());
@@ -932,8 +746,7 @@ struct GitInputScheme : InputScheme
bool exportIgnore = getExportIgnoreAttr(input); bool exportIgnore = getExportIgnoreAttr(input);
bool smudgeLfs = getLfsAttr(input); bool smudgeLfs = getLfsAttr(input);
auto accessor = repo->getAccessor( auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»", smudgeLfs);
rev, {.exportIgnore = exportIgnore, .smudgeLfs = smudgeLfs}, "«" + input.to_string() + "»");
/* If the repo has submodules, fetch them and return a mounted /* If the repo has submodules, fetch them and return a mounted
input accessor consisting of the accessor for the top-level input accessor consisting of the accessor for the top-level
@@ -968,8 +781,8 @@ struct GitInputScheme : InputScheme
attrs.insert_or_assign("submodules", Explicit<bool>{true}); attrs.insert_or_assign("submodules", Explicit<bool>{true});
attrs.insert_or_assign("lfs", Explicit<bool>{smudgeLfs}); attrs.insert_or_assign("lfs", Explicit<bool>{smudgeLfs});
attrs.insert_or_assign("allRefs", Explicit<bool>{true}); attrs.insert_or_assign("allRefs", Explicit<bool>{true});
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
mounts.insert_or_assign(submodule.path, submoduleAccessor); mounts.insert_or_assign(submodule.path, submoduleAccessor);
} }
@@ -986,7 +799,7 @@ struct GitInputScheme : InputScheme
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input>
getAccessorFromWorkdir(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const getAccessorFromWorkdir(ref<Store> store, RepoInfo & repoInfo, Input && input) const
{ {
auto repoPath = repoInfo.getPath().value(); auto repoPath = repoInfo.getPath().value();
@@ -1000,7 +813,7 @@ struct GitInputScheme : InputScheme
auto exportIgnore = getExportIgnoreAttr(input); auto exportIgnore = getExportIgnoreAttr(input);
ref<SourceAccessor> accessor = ref<SourceAccessor> accessor =
repo->getAccessor(repoInfo.workdirInfo, {.exportIgnore = exportIgnore}, makeNotAllowedError(repoPath)); repo->getAccessor(repoInfo.workdirInfo, exportIgnore, makeNotAllowedError(repoPath));
/* If the repo has submodules, return a mounted input accessor /* If the repo has submodules, return a mounted input accessor
consisting of the accessor for the top-level repo and the consisting of the accessor for the top-level repo and the
@@ -1018,8 +831,8 @@ struct GitInputScheme : InputScheme
// TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out
// attrs.insert_or_assign("allRefs", Explicit<bool>{ true }); // attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
auto submoduleInput = fetchers::Input::fromAttrs(settings, std::move(attrs)); auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs));
auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(settings, store); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store);
submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
/* If the submodule is dirty, mark this repo dirty as /* If the submodule is dirty, mark this repo dirty as
@@ -1046,12 +859,12 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("rev", rev.gitRev());
if (!getShallowAttr(input)) { if (!getShallowAttr(input)) {
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"revCount", rev == nullRev ? 0 : getRevCount(settings, repoInfo, repoPath, rev)); "revCount", rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev));
} }
verifyCommit(input, repo); verifyCommit(input, repo);
} else { } else {
repoInfo.warnDirty(settings); repoInfo.warnDirty(*input.settings);
if (repoInfo.workdirInfo.headRev) { if (repoInfo.workdirInfo.headRev) {
input.attrs.insert_or_assign("dirtyRev", repoInfo.workdirInfo.headRev->gitRev() + "-dirty"); input.attrs.insert_or_assign("dirtyRev", repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
@@ -1063,14 +876,14 @@ struct GitInputScheme : InputScheme
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"lastModified", "lastModified",
repoInfo.workdirInfo.headRev ? getLastModified(settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev) repoInfo.workdirInfo.headRev
: 0); ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev)
: 0);
return {accessor, std::move(input)}; return {accessor, std::move(input)};
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
@@ -1086,13 +899,13 @@ struct GitInputScheme : InputScheme
} }
auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath() auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath()
? getAccessorFromCommit(settings, store, repoInfo, std::move(input)) ? getAccessorFromCommit(store, repoInfo, std::move(input))
: getAccessorFromWorkdir(settings, store, repoInfo, std::move(input)); : getAccessorFromWorkdir(store, repoInfo, std::move(input));
return {accessor, std::move(final)}; return {accessor, std::move(final)};
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
auto makeFingerprint = [&](const Hash & rev) { auto makeFingerprint = [&](const Hash & rev) {
return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "") return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")
@@ -1123,7 +936,7 @@ struct GitInputScheme : InputScheme
} }
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
auto rev = input.getRev(); auto rev = input.getRev();
return rev && rev != nullRev; return rev && rev != nullRev;
+63 -109
View File
@@ -48,7 +48,7 @@ struct GitArchiveInputScheme : InputScheme
auto size = path.size(); auto size = path.size();
if (size == 3) { if (size == 3) {
if (std::regex_match(path[2], revRegex)) if (std::regex_match(path[2], revRegex))
rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); rev = parseGitHash(path[2]);
else if (isLegalRefName(path[2])) else if (isLegalRefName(path[2]))
ref = path[2]; ref = path[2];
else else
@@ -74,7 +74,7 @@ struct GitArchiveInputScheme : InputScheme
if (name == "rev") { if (name == "rev") {
if (rev) if (rev)
throw BadURL("URL '%s' contains multiple commit hashes", url); throw BadURL("URL '%s' contains multiple commit hashes", url);
rev = Hash::parseAny(value, HashAlgorithm::SHA1); rev = parseGitHash(value);
} else if (name == "ref") { } else if (name == "ref") {
if (!isLegalRefName(value)) if (!isLegalRefName(value))
throw BadURL("URL '%s' contains an invalid branch/tag name", url); throw BadURL("URL '%s' contains an invalid branch/tag name", url);
@@ -92,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme
if (ref && rev) if (ref && rev)
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev()); throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev());
Input input{}; Input input{settings};
input.attrs.insert_or_assign("type", std::string{schemeName()}); input.attrs.insert_or_assign("type", std::string{schemeName()});
input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("owner", path[0]);
input.attrs.insert_or_assign("repo", path[1]); input.attrs.insert_or_assign("repo", path[1]);
@@ -110,43 +110,18 @@ struct GitArchiveInputScheme : InputScheme
return input; return input;
} }
const std::map<std::string, AttributeInfo> & allowedAttrs() const override StringSet allowedAttrs() const override
{ {
static const std::map<std::string, AttributeInfo> attrs = { return {
{ "owner",
"owner", "repo",
{}, "ref",
}, "rev",
{ "narHash",
"repo", "lastModified",
{}, "host",
}, "treeHash",
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
{
"lastModified",
{},
},
{
"host",
{},
},
{
"treeHash",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override
@@ -154,7 +129,7 @@ struct GitArchiveInputScheme : InputScheme
getStrAttr(attrs, "owner"); getStrAttr(attrs, "owner");
getStrAttr(attrs, "repo"); getStrAttr(attrs, "repo");
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -258,9 +233,9 @@ struct GitArchiveInputScheme : InputScheme
std::optional<Hash> treeHash; std::optional<Hash> treeHash;
}; };
virtual RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const = 0; virtual RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
struct TarballInfo struct TarballInfo
{ {
@@ -268,7 +243,7 @@ struct GitArchiveInputScheme : InputScheme
time_t lastModified; time_t lastModified;
}; };
std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, Store & store, Input input) const std::pair<Input, TarballInfo> downloadArchive(ref<Store> store, Input input) const
{ {
if (!maybeGetStrAttr(input.attrs, "ref")) if (!maybeGetStrAttr(input.attrs, "ref"))
input.attrs.insert_or_assign("ref", "HEAD"); input.attrs.insert_or_assign("ref", "HEAD");
@@ -277,7 +252,7 @@ struct GitArchiveInputScheme : InputScheme
auto rev = input.getRev(); auto rev = input.getRev();
if (!rev) { if (!rev) {
auto refInfo = getRevFromRef(settings, store, input); auto refInfo = getRevFromRef(store, input);
rev = refInfo.rev; rev = refInfo.rev;
upstreamTreeHash = refInfo.treeHash; upstreamTreeHash = refInfo.treeHash;
debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev()); debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev());
@@ -286,7 +261,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.erase("ref"); input.attrs.erase("ref");
input.attrs.insert_or_assign("rev", rev->gitRev()); input.attrs.insert_or_assign("rev", rev->gitRev());
auto cache = settings.getCache(); auto cache = input.settings->getCache();
Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}};
Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}};
@@ -295,7 +270,7 @@ struct GitArchiveInputScheme : InputScheme
if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
if (settings.getTarballCache()->hasObject(treeHash)) if (getTarballCache()->hasObject(treeHash))
return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}; return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}};
else else
debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
@@ -303,7 +278,7 @@ struct GitArchiveInputScheme : InputScheme
} }
/* Stream the tarball into the tarball cache. */ /* Stream the tarball into the tarball cache. */
auto url = getDownloadUrl(settings, input); auto url = getDownloadUrl(input);
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
FileTransferRequest req(url.url); FileTransferRequest req(url.url);
@@ -315,7 +290,7 @@ struct GitArchiveInputScheme : InputScheme
*logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string())); *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string()));
TarArchive archive{*source}; TarArchive archive{*source};
auto tarballCache = settings.getTarballCache(); auto tarballCache = getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink(); auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink); auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush(); auto tree = parseSink->flush();
@@ -340,29 +315,28 @@ struct GitArchiveInputScheme : InputScheme
return {std::move(input), tarballInfo}; return {std::move(input), tarballInfo};
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
auto [input, tarballInfo] = downloadArchive(settings, store, _input); auto [input, tarballInfo] = downloadArchive(store, _input);
#if 0 #if 0
input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev());
#endif #endif
input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
auto accessor = auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
settings.getTarballCache()->getAccessor(tarballInfo.treeHash, {}, "«" + input.to_string() + "»");
return {accessor, input}; return {accessor, input};
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
/* Since we can't verify the integrity of the tarball from the /* Since we can't verify the integrity of the tarball from the
Git revision alone, we also require a NAR hash for Git revision alone, we also require a NAR hash for
locking. FIXME: in the future, we may want to require a Git locking. FIXME: in the future, we may want to require a Git
tree hash instead of a NAR hash. */ tree hash instead of a NAR hash. */
return input.getRev().has_value() && (settings.trustTarballsFromGitForges || input.getNarHash().has_value()); return input.getRev().has_value()
&& (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value());
} }
std::optional<ExperimentalFeature> experimentalFeature() const override std::optional<ExperimentalFeature> experimentalFeature() const override
@@ -370,7 +344,7 @@ struct GitArchiveInputScheme : InputScheme
return Xp::Flakes; return Xp::Flakes;
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto rev = input.getRev()) if (auto rev = input.getRev())
return rev->gitRev(); return rev->gitRev();
@@ -386,12 +360,6 @@ struct GitHubInputScheme : GitArchiveInputScheme
return "github"; return "github";
} }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
// Github supports PAT/OAuth2 tokens and HTTP Basic // Github supports PAT/OAuth2 tokens and HTTP Basic
@@ -418,7 +386,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
return getStrAttr(input.attrs, "repo"); return getStrAttr(input.attrs, "repo");
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
auto host = getHost(input); auto host = getHost(input);
auto url = fmt( auto url = fmt(
@@ -428,22 +396,22 @@ struct GitHubInputScheme : GitArchiveInputScheme
getRepo(input), getRepo(input),
*input.getRef()); *input.getRef());
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
auto downloadResult = downloadFile(store, settings, url, "source", headers); auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
return RefInfo{ return RefInfo{
.rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1), .rev = parseGitHash(std::string{json["sha"]}),
.treeHash = Hash::parseAny(std::string{json["commit"]["tree"]["sha"]}, HashAlgorithm::SHA1)}; .treeHash = parseGitHash(std::string{json["commit"]["tree"]["sha"]})};
} }
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
auto host = getHost(input); auto host = getHost(input);
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
// If we have no auth headers then we default to the public archive // If we have no auth headers then we default to the public archive
// urls so we do not run into rate limits. // urls so we do not run into rate limits.
@@ -457,13 +425,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto host = getHost(input); auto host = getHost(input);
Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(destDir);
} }
}; };
@@ -474,12 +441,6 @@ struct GitLabInputScheme : GitArchiveInputScheme
return "gitlab"; return "gitlab";
} }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
// Gitlab supports 4 kinds of authorization, two of which are // Gitlab supports 4 kinds of authorization, two of which are
@@ -499,7 +460,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1)); return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1));
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// See rate limiting note below // See rate limiting note below
@@ -510,14 +471,14 @@ struct GitLabInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"), getStrAttr(input.attrs, "repo"),
*input.getRef()); *input.getRef());
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
auto downloadResult = downloadFile(store, settings, url, "source", headers); auto downloadResult = downloadFile(store, *input.settings, url, "source", headers);
auto json = nlohmann::json::parse( auto json = nlohmann::json::parse(
store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root)); store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)}; return RefInfo{.rev = parseGitHash(std::string(json[0]["id"]))};
} }
if (json.is_array() && json.size() == 0) { if (json.is_array() && json.size() == 0) {
throw Error("No commits returned by GitLab API -- does the git ref really exist?"); throw Error("No commits returned by GitLab API -- does the git ref really exist?");
@@ -526,7 +487,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
} }
} }
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
// This endpoint has a rate limit threshold that may be // This endpoint has a rate limit threshold that may be
// server-specific and vary based whether the user is // server-specific and vary based whether the user is
@@ -541,20 +502,19 @@ struct GitLabInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(HashFormat::Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere // FIXME: get username somewhere
Input::fromURL( Input::fromURL(
settings, *input.settings,
fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(destDir);
} }
}; };
@@ -565,12 +525,6 @@ struct SourceHutInputScheme : GitArchiveInputScheme
return "sourcehut"; return "sourcehut";
} }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{ {
// SourceHut supports both PAT and OAuth2. See // SourceHut supports both PAT and OAuth2. See
@@ -581,7 +535,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
// Once it is implemented, however, should work as expected. // Once it is implemented, however, should work as expected.
} }
RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override RefInfo getRevFromRef(nix::ref<Store> store, const Input & input) const override
{ {
// TODO: In the future, when the sourcehut graphql API is implemented for mercurial // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
// and with anonymous access, this method should use it instead. // and with anonymous access, this method should use it instead.
@@ -592,12 +546,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
auto base_url = auto base_url =
fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
std::string refUri; std::string refUri;
if (ref == "HEAD") { if (ref == "HEAD") {
auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers); auto downloadFileResult = downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers);
auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
auto remoteLine = git::parseLsRemoteLine(getLine(contents).first); auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
if (!remoteLine) { if (!remoteLine) {
@@ -609,8 +563,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme
} }
std::regex refRegex(refUri); std::regex refRegex(refUri);
auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers); auto downloadFileResult =
auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root); downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers);
auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
std::istringstream is(contents); std::istringstream is(contents);
std::string line; std::string line;
@@ -624,10 +579,10 @@ struct SourceHutInputScheme : GitArchiveInputScheme
if (!id) if (!id)
throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref);
return RefInfo{.rev = Hash::parseAny(*id, HashAlgorithm::SHA1)}; return RefInfo{.rev = parseGitHash(*id)};
} }
DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const override DownloadUrl getDownloadUrl(const Input & input) const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto url = auto url =
@@ -637,19 +592,18 @@ struct SourceHutInputScheme : GitArchiveInputScheme
getStrAttr(input.attrs, "repo"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(HashFormat::Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(settings, host, input); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
return DownloadUrl{parseURL(url), headers}; return DownloadUrl{parseURL(url), headers};
} }
void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) void clone(const Input & input, const Path & destDir) const override
const override
{ {
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
Input::fromURL( Input::fromURL(
settings, *input.settings,
fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef(), input.getRev()) .applyOverrides(input.getRef(), input.getRev())
.clone(settings, store, destDir); .clone(destDir);
} }
}; };
@@ -11,12 +11,6 @@
#include <sys/types.h> #include <sys/types.h>
namespace nix {
struct GitRepo;
}
namespace nix::fetchers { namespace nix::fetchers {
struct Cache; struct Cache;
@@ -131,8 +125,6 @@ struct Settings : public Config
ref<Cache> getCache() const; ref<Cache> getCache() const;
ref<GitRepo> getTarballCache() const;
private: private:
mutable Sync<std::shared_ptr<Cache>> _cache; mutable Sync<std::shared_ptr<Cache>> _cache;
}; };
@@ -36,6 +36,13 @@ struct Input
{ {
friend struct InputScheme; friend struct InputScheme;
const Settings * settings;
Input(const Settings & settings)
: settings{&settings}
{
}
std::shared_ptr<InputScheme> scheme; // note: can be null std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs; Attrs attrs;
@@ -80,7 +87,7 @@ public:
* attributes like a Git revision or NAR hash that uniquely * attributes like a Git revision or NAR hash that uniquely
* identify its contents. * identify its contents.
*/ */
bool isLocked(const Settings & settings) const; bool isLocked() const;
/** /**
* Only for relative path flakes, i.e. 'path:./foo', returns the * Only for relative path flakes, i.e. 'path:./foo', returns the
@@ -113,7 +120,7 @@ public:
* Fetch the entire input into the Nix store, returning the * Fetch the entire input into the Nix store, returning the
* location in the Nix store and the locked input. * location in the Nix store and the locked input.
*/ */
std::pair<StorePath, Input> fetchToStore(const Settings & settings, Store & store) const; std::pair<StorePath, Input> fetchToStore(ref<Store> store) const;
/** /**
* Check the locking attributes in `result` against * Check the locking attributes in `result` against
@@ -133,17 +140,17 @@ public:
* input without copying it to the store. Also return a possibly * input without copying it to the store. Also return a possibly
* unlocked input. * unlocked input.
*/ */
std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, Store & store) const; std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store) const;
private: private:
std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, Store & store) const; std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(ref<Store> store) const;
public: public:
Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const; Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const;
void clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const; void clone(const Path & destDir) const;
std::optional<std::filesystem::path> getSourcePath() const; std::optional<std::filesystem::path> getSourcePath() const;
@@ -173,7 +180,7 @@ public:
* *
* This is not a stable identifier between Nix versions, but not guaranteed to change either. * This is not a stable identifier between Nix versions, but not guaranteed to change either.
*/ */
std::optional<std::string> getFingerprint(Store & store) const; std::optional<std::string> getFingerprint(ref<Store> store) const;
}; };
/** /**
@@ -203,34 +210,20 @@ struct InputScheme
*/ */
virtual std::string_view schemeName() const = 0; virtual std::string_view schemeName() const = 0;
/**
* Longform description of this scheme, for documentation purposes.
*/
virtual std::string schemeDescription() const = 0;
// TODO remove these defaults
struct AttributeInfo
{
const char * type = "String";
bool required = true;
const char * doc = "";
};
/** /**
* Allowed attributes in an attribute set that is converted to an * Allowed attributes in an attribute set that is converted to an
* input, and documentation for each attribute. * input.
* *
* `type` is not included from this map, because the `type` field is * `type` is not included from this set, because the `type` field is
parsed first to choose which scheme; `type` is always required. parsed first to choose which scheme; `type` is always required.
*/ */
virtual const std::map<std::string, AttributeInfo> & allowedAttrs() const = 0; virtual StringSet allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const; virtual ParsedURL toURL(const Input & input) const;
virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const; virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const;
virtual void virtual void clone(const Input & input, const Path & destDir) const;
clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const;
virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const; virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
@@ -240,8 +233,7 @@ struct InputScheme
std::string_view contents, std::string_view contents,
std::optional<std::string> commitMsg) const; std::optional<std::string> commitMsg) const;
virtual std::pair<ref<SourceAccessor>, Input> virtual std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const = 0;
getAccessor(const Settings & settings, Store & store, const Input & input) const = 0;
/** /**
* Is this `InputScheme` part of an experimental feature? * Is this `InputScheme` part of an experimental feature?
@@ -253,12 +245,12 @@ struct InputScheme
return true; return true;
} }
virtual std::optional<std::string> getFingerprint(Store & store, const Input & input) const virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
{ {
return std::nullopt; return std::nullopt;
} }
virtual bool isLocked(const Settings & settings, const Input & input) const virtual bool isLocked(const Input & input) const
{ {
return false; return false;
} }
@@ -277,12 +269,7 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>; nlohmann::json dumpRegisterInputSchemeInfo();
/**
* Use this for docs, not for finding a specific scheme
*/
const InputSchemeMap & getAllInputSchemes();
struct PublicKey struct PublicKey
{ {
@@ -5,7 +5,7 @@
#include "nix/util/serialise.hh" #include "nix/util/serialise.hh"
#include "nix/util/url.hh" #include "nix/util/url.hh"
#include <git2/repository.h> #include <git2-experimental/repository.h>
#include <nlohmann/json_fwd.hpp> #include <nlohmann/json_fwd.hpp>
@@ -22,12 +22,6 @@ struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
virtual Hash flush() = 0; virtual Hash flush() = 0;
}; };
struct GitAccessorOptions
{
bool exportIgnore = false;
bool smudgeLfs = false;
};
struct GitRepo struct GitRepo
{ {
virtual ~GitRepo() {} virtual ~GitRepo() {}
@@ -95,10 +89,10 @@ struct GitRepo
virtual bool hasObject(const Hash & oid) = 0; virtual bool hasObject(const Hash & oid) = 0;
virtual ref<SourceAccessor> virtual ref<SourceAccessor>
getAccessor(const Hash & rev, const GitAccessorOptions & options, std::string displayPrefix) = 0; getAccessor(const Hash & rev, bool exportIgnore, std::string displayPrefix, bool smudgeLfs = false) = 0;
virtual ref<SourceAccessor> getAccessor( virtual ref<SourceAccessor>
const WorkdirInfo & wd, const GitAccessorOptions & options, MakeNotAllowedError makeNotAllowedError) = 0; getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0; virtual ref<GitFileSystemObjectSink> getFileSystemObjectSink() = 0;
@@ -126,6 +120,8 @@ struct GitRepo
virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0;
}; };
ref<GitRepo> getTarballCache();
// A helper to ensure that the `git_*_free` functions get called. // A helper to ensure that the `git_*_free` functions get called.
template<auto del> template<auto del>
struct Deleter struct Deleter
@@ -171,4 +167,14 @@ struct Setter
*/ */
bool isLegalRefName(const std::string & refName); bool isLegalRefName(const std::string & refName);
/**
* Parse a base16-encoded git hash string and determine the hash
* algorithm based on the length (40 chars = SHA1, 64 chars = SHA256).
*
* @note For Nix-native information we should *not* do length tricks,
* but instead always rely on an explicit algorithm. This hack should be
* only for foreign hash literals.
*/
Hash parseGitHash(std::string_view hashStr);
} // namespace nix } // namespace nix
@@ -3,7 +3,6 @@
namespace nix::fetchers { namespace nix::fetchers {
enum class UseRegistries : int; enum class UseRegistries : int;
struct Settings;
struct InputCache struct InputCache
{ {
@@ -15,8 +14,7 @@ struct InputCache
Attrs extraAttrs; Attrs extraAttrs;
}; };
CachedResult CachedResult getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
getAccessor(const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries);
struct CachedInput struct CachedInput
{ {
@@ -13,6 +13,8 @@ namespace nix::fetchers {
struct Registry struct Registry
{ {
const Settings & settings;
enum RegistryType { enum RegistryType {
Flag = 0, Flag = 0,
User = 1, User = 1,
@@ -32,8 +34,9 @@ struct Registry
std::vector<Entry> entries; std::vector<Entry> entries;
Registry(RegistryType type) Registry(const Settings & settings, RegistryType type)
: type{type} : settings{settings}
, type{type}
{ {
} }
@@ -54,7 +57,7 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
Path getUserRegistryPath(); Path getUserRegistryPath();
Registries getRegistries(const Settings & settings, Store & store); Registries getRegistries(const Settings & settings, ref<Store> store);
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs); void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs);
@@ -68,7 +71,6 @@ enum class UseRegistries : int {
* Rewrite a flakeref using the registries. If `filter` is set, only * Rewrite a flakeref using the registries. If `filter` is set, only
* use the registries for which the filter function returns true. * use the registries for which the filter function returns true.
*/ */
std::pair<Input, Attrs> std::pair<Input, Attrs> lookupInRegistries(ref<Store> store, const Input & input, UseRegistries useRegistries);
lookupInRegistries(const Settings & settings, Store & store, const Input & input, UseRegistries useRegistries);
} // namespace nix::fetchers } // namespace nix::fetchers
@@ -25,7 +25,7 @@ struct DownloadFileResult
}; };
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
Store & store, ref<Store> store,
const Settings & settings, const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@@ -43,6 +43,6 @@ struct DownloadTarballResult
* Download and import a tarball into the Git cache. The result is the * Download and import a tarball into the Git cache. The result is the
* Git tree hash of the root directory. * Git tree hash of the root directory.
*/ */
ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url); ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url);
} // namespace nix::fetchers } // namespace nix::fetchers
+11 -31
View File
@@ -23,7 +23,7 @@ struct IndirectInputScheme : InputScheme
if (path.size() == 1) { if (path.size() == 1) {
} else if (path.size() == 2) { } else if (path.size() == 2) {
if (std::regex_match(path[1], revRegex)) if (std::regex_match(path[1], revRegex))
rev = Hash::parseAny(path[1], HashAlgorithm::SHA1); rev = parseGitHash(path[1]);
else if (isLegalRefName(path[1])) else if (isLegalRefName(path[1]))
ref = path[1]; ref = path[1];
else else
@@ -34,7 +34,7 @@ struct IndirectInputScheme : InputScheme
ref = path[1]; ref = path[1];
if (!std::regex_match(path[2], revRegex)) if (!std::regex_match(path[2], revRegex))
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]); throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]);
rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); rev = parseGitHash(path[2]);
} else } else
throw BadURL("GitHub URL '%s' is invalid", url); throw BadURL("GitHub URL '%s' is invalid", url);
@@ -44,7 +44,7 @@ struct IndirectInputScheme : InputScheme
// FIXME: forbid query params? // FIXME: forbid query params?
Input input{}; Input input{settings};
input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id); input.attrs.insert_or_assign("id", id);
if (rev) if (rev)
@@ -60,33 +60,14 @@ struct IndirectInputScheme : InputScheme
return "indirect"; return "indirect";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
// TODO return {
return ""; "id",
} "ref",
"rev",
const std::map<std::string, AttributeInfo> & allowedAttrs() const override "narHash",
{
static const std::map<std::string, AttributeInfo> attrs = {
{
"id",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -95,7 +76,7 @@ struct IndirectInputScheme : InputScheme
if (!std::regex_match(id, flakeRegex)) if (!std::regex_match(id, flakeRegex))
throw BadURL("'%s' is not a valid flake ID", id); throw BadURL("'%s' is not a valid flake ID", id);
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -125,8 +106,7 @@ struct IndirectInputScheme : InputScheme
return input; return input;
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const override
getAccessor(const Settings & settings, Store & store, const Input & input) const override
{ {
throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
} }
+5 -5
View File
@@ -5,23 +5,23 @@
namespace nix::fetchers { namespace nix::fetchers {
InputCache::CachedResult InputCache::getAccessor( InputCache::CachedResult
const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries) InputCache::getAccessor(ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
{ {
auto fetched = lookup(originalInput); auto fetched = lookup(originalInput);
Input resolvedInput = originalInput; Input resolvedInput = originalInput;
if (!fetched) { if (!fetched) {
if (originalInput.isDirect()) { if (originalInput.isDirect()) {
auto [accessor, lockedInput] = originalInput.getAccessor(settings, store); auto [accessor, lockedInput] = originalInput.getAccessor(store);
fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor});
} else { } else {
if (useRegistries != UseRegistries::No) { if (useRegistries != UseRegistries::No) {
auto [res, extraAttrs] = lookupInRegistries(settings, store, originalInput, useRegistries); auto [res, extraAttrs] = lookupInRegistries(store, originalInput, useRegistries);
resolvedInput = std::move(res); resolvedInput = std::move(res);
fetched = lookup(resolvedInput); fetched = lookup(resolvedInput);
if (!fetched) { if (!fetched) {
auto [accessor, lockedInput] = resolvedInput.getAccessor(settings, store); auto [accessor, lockedInput] = resolvedInput.getAccessor(store);
fetched.emplace( fetched.emplace(
CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs}); CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs});
} }
+25 -51
View File
@@ -68,41 +68,16 @@ struct MercurialInputScheme : InputScheme
return "hg"; return "hg";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
// TODO return {
return ""; "url",
} "ref",
"rev",
const std::map<std::string, AttributeInfo> & allowedAttrs() const override "revCount",
{ "narHash",
static const std::map<std::string, AttributeInfo> attrs = { "name",
{
"url",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"narHash",
{},
},
{
"name",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -114,7 +89,7 @@ struct MercurialInputScheme : InputScheme
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
} }
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -179,7 +154,7 @@ struct MercurialInputScheme : InputScheme
return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()}; return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()};
} }
StorePath fetchToStore(const Settings & settings, Store & store, Input & input) const StorePath fetchToStore(ref<Store> store, Input & input) const
{ {
auto origRev = input.getRev(); auto origRev = input.getRev();
@@ -201,10 +176,10 @@ struct MercurialInputScheme : InputScheme
/* This is an unclean working tree. So copy all tracked /* This is an unclean working tree. So copy all tracked
files. */ files. */
if (!settings.allowDirty) if (!input.settings->allowDirty)
throw Error("Mercurial tree '%s' is unclean", actualUrl); throw Error("Mercurial tree '%s' is unclean", actualUrl);
if (settings.warnDirty) if (input.settings->warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl); warn("Mercurial tree '%s' is unclean", actualUrl);
input.attrs.insert_or_assign("ref", chomp(runHg({"branch", "-R", actualUrl}))); input.attrs.insert_or_assign("ref", chomp(runHg({"branch", "-R", actualUrl})));
@@ -230,7 +205,7 @@ struct MercurialInputScheme : InputScheme
return files.count(file); return files.count(file);
}; };
auto storePath = store.addToStore( auto storePath = store->addToStore(
input.getName(), input.getName(),
{getFSSourceAccessor(), CanonPath(actualPath)}, {getFSSourceAccessor(), CanonPath(actualPath)},
ContentAddressMethod::Raw::NixArchive, ContentAddressMethod::Raw::NixArchive,
@@ -251,7 +226,7 @@ struct MercurialInputScheme : InputScheme
"Hash '%s' is not supported by Mercurial. Only sha1 is supported.", "Hash '%s' is not supported by Mercurial. Only sha1 is supported.",
rev.to_string(HashFormat::Base16, true)); rev.to_string(HashFormat::Base16, true));
return Cache::Key{"hgRev", {{"store", store.storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}}; return Cache::Key{"hgRev", {{"store", store->storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
}; };
auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath { auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath {
@@ -265,13 +240,13 @@ struct MercurialInputScheme : InputScheme
Cache::Key refToRevKey{"hgRefToRev", {{"url", actualUrl}, {"ref", *input.getRef()}}}; Cache::Key refToRevKey{"hgRefToRev", {{"url", actualUrl}, {"ref", *input.getRef()}}};
if (!input.getRev()) { if (!input.getRev()) {
if (auto res = settings.getCache()->lookupWithTTL(refToRevKey)) if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey))
input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev());
} }
/* If we have a rev, check if we have a cached store path. */ /* If we have a rev, check if we have a cached store path. */
if (auto rev = input.getRev()) { if (auto rev = input.getRev()) {
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), store)) if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
} }
@@ -325,7 +300,7 @@ struct MercurialInputScheme : InputScheme
/* Now that we have the rev, check the cache again for a /* Now that we have the rev, check the cache again for a
cached store path. */ cached store path. */
if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), store)) if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store))
return makeResult(res->value, res->storePath); return makeResult(res->value, res->storePath);
Path tmpDir = createTempDir(); Path tmpDir = createTempDir();
@@ -335,39 +310,38 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir + "/.hg_archival.txt"); deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)}); auto storePath = store->addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
Attrs infoAttrs({ Attrs infoAttrs({
{"revCount", (uint64_t) revCount}, {"revCount", (uint64_t) revCount},
}); });
if (!origRev) if (!origRev)
settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
settings.getCache()->upsert(revInfoKey(rev), store, infoAttrs, storePath); input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
return makeResult(infoAttrs, std::move(storePath)); return makeResult(infoAttrs, std::move(storePath));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
auto storePath = fetchToStore(settings, store, input); auto storePath = fetchToStore(store, input);
auto accessor = store.requireStoreObjectAccessor(storePath); auto accessor = store->requireStoreObjectAccessor(storePath);
accessor->setPathDisplay("«" + input.to_string() + "»"); accessor->setPathDisplay("«" + input.to_string() + "»");
return {accessor, input}; return {accessor, input};
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
return (bool) input.getRev(); return (bool) input.getRev();
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto rev = input.getRev()) if (auto rev = input.getRev())
return rev->gitRev(); return rev->gitRev();
+1 -1
View File
@@ -28,7 +28,7 @@ subdir('nix-meson-build-support/subprojects')
nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
deps_public += nlohmann_json deps_public += nlohmann_json
libgit2 = dependency('libgit2', version : '>= 1.9') libgit2 = dependency('libgit2-experimental', version : '>= 1.9')
deps_private += libgit2 deps_private += libgit2
subdir('nix-meson-build-support/common') subdir('nix-meson-build-support/common')
+20 -43
View File
@@ -17,7 +17,7 @@ struct PathInputScheme : InputScheme
if (url.authority && url.authority->host.size()) if (url.authority && url.authority->host.size())
throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority); throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority);
Input input{}; Input input{settings};
input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("type", "path");
input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path)); input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path));
@@ -40,49 +40,27 @@ struct PathInputScheme : InputScheme
return "path"; return "path";
} }
std::string schemeDescription() const override StringSet allowedAttrs() const override
{ {
// TODO return {
return ""; "path",
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = {
{
"path",
{},
},
/* Allow the user to pass in "fake" tree info /* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g. the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/ */
{ "rev",
"rev", "revCount",
{}, "lastModified",
}, "narHash",
{
"revCount",
{},
},
{
"lastModified",
{},
},
{
"narHash",
{},
},
}; };
return attrs;
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
{ {
getStrAttr(attrs, "path"); getStrAttr(attrs, "path");
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@@ -123,7 +101,7 @@ struct PathInputScheme : InputScheme
return path; return path;
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
return (bool) input.getNarHash(); return (bool) input.getNarHash();
} }
@@ -138,8 +116,7 @@ struct PathInputScheme : InputScheme
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
Input input(_input); Input input(_input);
auto path = getStrAttr(input.attrs, "path"); auto path = getStrAttr(input.attrs, "path");
@@ -147,31 +124,31 @@ struct PathInputScheme : InputScheme
auto absPath = getAbsPath(input); auto absPath = getAbsPath(input);
// FIXME: check whether access to 'path' is allowed. // FIXME: check whether access to 'path' is allowed.
auto storePath = store.maybeParseStorePath(absPath.string()); auto storePath = store->maybeParseStorePath(absPath.string());
if (storePath) if (storePath)
store.addTempRoot(*storePath); store->addTempRoot(*storePath);
time_t mtime = 0; time_t mtime = 0;
if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) { if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
// FIXME: try to substitute storePath. // FIXME: try to substitute storePath.
auto src = sinkToSource( auto src = sinkToSource(
[&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
storePath = store.addToStoreFromDump(*src, "source"); storePath = store->addToStoreFromDump(*src, "source");
} }
auto accessor = store.requireStoreObjectAccessor(*storePath); auto accessor = store->requireStoreObjectAccessor(*storePath);
// To prevent `fetchToStore()` copying the path again to Nix // To prevent `fetchToStore()` copying the path again to Nix
// store, pre-create an entry in the fetcher cache. // store, pre-create an entry in the fetcher cache.
auto info = store.queryPathInfo(*storePath); auto info = store->queryPathInfo(*storePath);
accessor->fingerprint = accessor->fingerprint =
fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
settings.getCache()->upsert( input.settings->getCache()->upsert(
makeFetchToStoreCacheKey( makeFetchToStoreCacheKey(
input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
store, *store,
{}, {},
*storePath); *storePath);
+13 -14
View File
@@ -14,10 +14,10 @@ std::shared_ptr<Registry> Registry::read(const Settings & settings, const Source
{ {
debug("reading registry '%s'", path); debug("reading registry '%s'", path);
auto registry = std::make_shared<Registry>(type); auto registry = std::make_shared<Registry>(settings, type);
if (!path.pathExists()) if (!path.pathExists())
return std::make_shared<Registry>(type); return std::make_shared<Registry>(settings, type);
try { try {
@@ -125,23 +125,23 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
return customRegistry; return customRegistry;
} }
std::shared_ptr<Registry> getFlagRegistry() std::shared_ptr<Registry> getFlagRegistry(const Settings & settings)
{ {
static auto flagRegistry = std::make_shared<Registry>(Registry::Flag); static auto flagRegistry = std::make_shared<Registry>(settings, Registry::Flag);
return flagRegistry; return flagRegistry;
} }
void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs) void overrideRegistry(const Input & from, const Input & to, const Attrs & extraAttrs)
{ {
getFlagRegistry()->add(from, to, extraAttrs); getFlagRegistry(*from.settings)->add(from, to, extraAttrs);
} }
static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store) static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
{ {
static auto reg = [&]() { static auto reg = [&]() {
auto path = settings.flakeRegistry.get(); auto path = settings.flakeRegistry.get();
if (path == "") { if (path == "") {
return std::make_shared<Registry>(Registry::Global); // empty registry return std::make_shared<Registry>(settings, Registry::Global); // empty registry
} }
return Registry::read( return Registry::read(
@@ -149,9 +149,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
[&] -> SourcePath { [&] -> SourcePath {
if (!isAbsolute(path)) { if (!isAbsolute(path)) {
auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
if (auto store2 = dynamic_cast<LocalFSStore *>(&store)) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
return {store.requireStoreObjectAccessor(storePath)}; return {store->requireStoreObjectAccessor(storePath)};
} else { } else {
return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks(); return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
} }
@@ -162,18 +162,17 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
return reg; return reg;
} }
Registries getRegistries(const Settings & settings, Store & store) Registries getRegistries(const Settings & settings, ref<Store> store)
{ {
Registries registries; Registries registries;
registries.push_back(getFlagRegistry()); registries.push_back(getFlagRegistry(settings));
registries.push_back(getUserRegistry(settings)); registries.push_back(getUserRegistry(settings));
registries.push_back(getSystemRegistry(settings)); registries.push_back(getSystemRegistry(settings));
registries.push_back(getGlobalRegistry(settings, store)); registries.push_back(getGlobalRegistry(settings, store));
return registries; return registries;
} }
std::pair<Input, Attrs> std::pair<Input, Attrs> lookupInRegistries(ref<Store> store, const Input & _input, UseRegistries useRegistries)
lookupInRegistries(const Settings & settings, Store & store, const Input & _input, UseRegistries useRegistries)
{ {
Attrs extraAttrs; Attrs extraAttrs;
int n = 0; int n = 0;
@@ -188,7 +187,7 @@ restart:
if (n > 100) if (n > 100)
throw Error("cycle detected in flake registry for '%s'", input.to_string()); throw Error("cycle detected in flake registry for '%s'", input.to_string());
for (auto & registry : getRegistries(settings, store)) { for (auto & registry : getRegistries(*input.settings, store)) {
if (useRegistries == UseRegistries::Limited if (useRegistries == UseRegistries::Limited
&& !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global)) && !(registry->type == fetchers::Registry::Flag || registry->type == fetchers::Registry::Global))
continue; continue;
+34 -136
View File
@@ -13,7 +13,7 @@
namespace nix::fetchers { namespace nix::fetchers {
DownloadFileResult downloadFile( DownloadFileResult downloadFile(
Store & store, ref<Store> store,
const Settings & settings, const Settings & settings,
const std::string & url, const std::string & url,
const std::string & name, const std::string & name,
@@ -28,7 +28,7 @@ DownloadFileResult downloadFile(
{"name", name}, {"name", name},
}}}; }}};
auto cached = settings.getCache()->lookupStorePath(key, store); auto cached = settings.getCache()->lookupStorePath(key, *store);
auto useCached = [&]() -> DownloadFileResult { auto useCached = [&]() -> DownloadFileResult {
return { return {
@@ -74,7 +74,7 @@ DownloadFileResult downloadFile(
dumpString(res.data, sink); dumpString(res.data, sink);
auto hash = hashString(HashAlgorithm::SHA256, res.data); auto hash = hashString(HashAlgorithm::SHA256, res.data);
auto info = ValidPathInfo::makeFromCA( auto info = ValidPathInfo::makeFromCA(
store, *store,
name, name,
FixedOutputInfo{ FixedOutputInfo{
.method = FileIngestionMethod::Flat, .method = FileIngestionMethod::Flat,
@@ -84,7 +84,7 @@ DownloadFileResult downloadFile(
hashString(HashAlgorithm::SHA256, sink.s)); hashString(HashAlgorithm::SHA256, sink.s));
info.narSize = sink.s.size(); info.narSize = sink.s.size();
auto source = StringSource{sink.s}; auto source = StringSource{sink.s};
store.addToStore(info, source, NoRepair, NoCheckSigs); store->addToStore(info, source, NoRepair, NoCheckSigs);
storePath = std::move(info.path); storePath = std::move(info.path);
} }
@@ -93,7 +93,7 @@ DownloadFileResult downloadFile(
key.second.insert_or_assign("url", url); key.second.insert_or_assign("url", url);
assert(!res.urls.empty()); assert(!res.urls.empty());
infoAttrs.insert_or_assign("url", *res.urls.rbegin()); infoAttrs.insert_or_assign("url", *res.urls.rbegin());
settings.getCache()->upsert(key, store, infoAttrs, *storePath); settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
} }
return { return {
@@ -136,11 +136,11 @@ static DownloadTarballResult downloadTarball_(
.treeHash = treeHash, .treeHash = treeHash,
.lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
.accessor = settings.getTarballCache()->getAccessor(treeHash, {}, displayPrefix), .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix),
}; };
}; };
if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash")))
cached.reset(); cached.reset();
if (cached && !cached->expired) if (cached && !cached->expired)
@@ -179,7 +179,7 @@ static DownloadTarballResult downloadTarball_(
TarArchive{path}; TarArchive{path};
}) })
: TarArchive{*source}; : TarArchive{*source};
auto tarballCache = settings.getTarballCache(); auto tarballCache = getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink(); auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink); auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush(); auto tree = parseSink->flush();
@@ -214,7 +214,7 @@ static DownloadTarballResult downloadTarball_(
return attrsToResult(infoAttrs); return attrsToResult(infoAttrs);
} }
ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url) ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url)
{ {
/* Go through Input::getAccessor() to ensure that the resulting /* Go through Input::getAccessor() to ensure that the resulting
accessor has a fingerprint. */ accessor has a fingerprint. */
@@ -224,7 +224,7 @@ ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, co
auto input = Input::fromAttrs(settings, std::move(attrs)); auto input = Input::fromAttrs(settings, std::move(attrs));
return input.getAccessor(settings, store).first; return input.getAccessor(store).first;
} }
// An input scheme corresponding to a curl-downloadable resource. // An input scheme corresponding to a curl-downloadable resource.
@@ -252,7 +252,7 @@ struct CurlInputScheme : InputScheme
if (!isValidURL(_url, requireTree)) if (!isValidURL(_url, requireTree))
return std::nullopt; return std::nullopt;
Input input{}; Input input{settings};
auto url = _url; auto url = _url;
@@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
HTTP request. Now that we've processed the Nix-specific HTTP request. Now that we've processed the Nix-specific
attributes above, remove them so we don't also send them as attributes above, remove them so we don't also send them as
part of the HTTP request. */ part of the HTTP request. */
for (auto & [param, _] : allowedAttrs()) for (auto & param : allowedAttrs())
url.query.erase(param); url.query.erase(param);
input.attrs.insert_or_assign("type", std::string{schemeName()}); input.attrs.insert_or_assign("type", std::string{schemeName()});
@@ -286,88 +286,23 @@ struct CurlInputScheme : InputScheme
return input; return input;
} }
static const std::map<std::string, AttributeInfo> & allowedAttrsImpl() StringSet allowedAttrs() const override
{ {
static const std::map<std::string, AttributeInfo> attrs = { return {
{ "type",
"url", "url",
{ "narHash",
.type = "String", "name",
.required = true, "unpack",
.doc = R"( "rev",
Supported protocols: "revCount",
"lastModified",
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
)",
},
},
{
"narHash",
{},
},
{
"name",
{},
},
{
"unpack",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
}; };
return attrs;
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
return allowedAttrsImpl();
} }
std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
{ {
Input input{}; Input input{settings};
input.attrs = attrs; input.attrs = attrs;
// input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); // input.locked = (bool) maybeGetStrAttr(input.attrs, "hash");
@@ -384,7 +319,7 @@ struct CurlInputScheme : InputScheme
return url; return url;
} }
bool isLocked(const Settings & settings, const Input & input) const override bool isLocked(const Input & input) const override
{ {
return (bool) input.getNarHash(); return (bool) input.getNarHash();
} }
@@ -397,14 +332,6 @@ struct FileInputScheme : CurlInputScheme
return "file"; return "file";
} }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override bool isValidURL(const ParsedURL & url, bool requireTree) const override
{ {
auto parsedUrlScheme = parseUrlScheme(url.scheme); auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -413,8 +340,7 @@ struct FileInputScheme : CurlInputScheme
: (!requireTree && !hasTarballExtension(url))); : (!requireTree && !hasTarballExtension(url)));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
auto input(_input); auto input(_input);
@@ -422,12 +348,12 @@ struct FileInputScheme : CurlInputScheme
the Nix store directly, since there is little deduplication the Nix store directly, since there is little deduplication
benefit in using the Git cache for single big files like benefit in using the Git cache for single big files like
tarballs. */ tarballs. */
auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName()); auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName());
auto narHash = store.queryPathInfo(file.storePath)->narHash; auto narHash = store->queryPathInfo(file.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
auto accessor = ref{store.getFSAccessor(file.storePath)}; auto accessor = ref{store->getFSAccessor(file.storePath)};
accessor->setPathDisplay("«" + input.to_string() + "»"); accessor->setPathDisplay("«" + input.to_string() + "»");
@@ -442,34 +368,6 @@ struct TarballInputScheme : CurlInputScheme
return "tarball"; return "tarball";
} }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Download a tar archive and extract it into the Nix store.
This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
)");
}
const std::map<std::string, AttributeInfo> & allowedAttrs() const override
{
static const std::map<std::string, AttributeInfo> attrs = [] {
auto attrs = CurlInputScheme::allowedAttrsImpl();
// Override the "url" attribute to add tarball-specific example
attrs["url"].doc = R"(
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
)";
return attrs;
}();
return attrs;
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override bool isValidURL(const ParsedURL & url, bool requireTree) const override
{ {
auto parsedUrlScheme = parseUrlScheme(url.scheme); auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -479,15 +377,15 @@ struct TarballInputScheme : CurlInputScheme
: (requireTree || hasTarballExtension(url))); : (requireTree || hasTarballExtension(url)));
} }
std::pair<ref<SourceAccessor>, Input> std::pair<ref<SourceAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
{ {
auto input(_input); auto input(_input);
auto result = downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); auto result =
downloadTarball_(*input.settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»");
if (result.immutableUrl) { if (result.immutableUrl) {
auto immutableInput = Input::fromURL(settings, *result.immutableUrl); auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl);
// FIXME: would be nice to support arbitrary flakerefs // FIXME: would be nice to support arbitrary flakerefs
// here, e.g. git flakes. // here, e.g. git flakes.
if (immutableInput.getType() != "tarball") if (immutableInput.getType() != "tarball")
@@ -500,12 +398,12 @@ struct TarballInputScheme : CurlInputScheme
input.attrs.insert_or_assign( input.attrs.insert_or_assign(
"narHash", "narHash",
settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash).to_string(HashFormat::SRI, true)); getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
return {result.accessor, input}; return {result.accessor, input};
} }
std::optional<std::string> getFingerprint(Store & store, const Input & input) const override std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
{ {
if (auto narHash = input.getNarHash()) if (auto narHash = input.getNarHash())
return narHash->to_string(HashFormat::SRI, true); return narHash->to_string(HashFormat::SRI, true);
+1 -1
View File
@@ -200,7 +200,7 @@ nix_value * nix_locked_flake_get_output_attrs(
nix_clear_err(context); nix_clear_err(context);
try { try {
auto v = nix_alloc_value(context, evalState); auto v = nix_alloc_value(context, evalState);
nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, *v->value); nix::flake::callFlake(evalState->state, *lockedFlake->lockedFlake, v->value);
return v; return v;
} }
NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_NULL
+4 -1
View File
@@ -10,6 +10,9 @@ lockFileStr:
# unlocked trees. # unlocked trees.
overrides: overrides:
# This is `prim_fetchFinalTree`.
fetchTreeFinal:
let let
inherit (builtins) mapAttrs; inherit (builtins) mapAttrs;
@@ -49,7 +52,7 @@ let
else else
# FIXME: remove obsolete node.info. # FIXME: remove obsolete node.info.
# Note: lock file entries are always final. # Note: lock file entries are always final.
builtins.fetchFinalTree (node.info or { } // removeAttrs node.locked [ "dir" ]); fetchTreeFinal (node.info or { } // removeAttrs node.locked [ "dir" ]);
subdir = overrides.${key}.dir or node.locked.dir or ""; subdir = overrides.${key}.dir or node.locked.dir or "";
+3 -3
View File
@@ -38,7 +38,7 @@ PrimOp getFlake(const Settings & settings)
std::string flakeRefS( std::string flakeRefS(
state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake"));
auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); auto flakeRef = nix::parseFlakeRef(state.fetchSettings, flakeRefS, {}, true);
if (state.settings.pureEval && !flakeRef.input.isLocked(state.fetchSettings)) if (state.settings.pureEval && !flakeRef.input.isLocked())
throw Error( throw Error(
"cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", "cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)",
flakeRefS, flakeRefS,
@@ -93,7 +93,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
auto & vv = binds.alloc(s); auto & vv = binds.alloc(s);
std::visit( std::visit(
overloaded{ overloaded{
[&vv, &state](const std::string & value) { vv.mkString(value, state.mem); }, [&vv](const std::string & value) { vv.mkString(value); },
[&vv](const uint64_t & value) { vv.mkInt(value); }, [&vv](const uint64_t & value) { vv.mkInt(value); },
[&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }}, [&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }},
value); value);
@@ -156,7 +156,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
} }
} }
auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs);
v.mkString(flakeRef.to_string(), state.mem); v.mkString(flakeRef.to_string());
} }
nix::PrimOp flakeRefToString({ nix::PrimOp flakeRefToString({
+12 -12
View File
@@ -372,8 +372,7 @@ static Flake getFlake(
const InputAttrPath & lockRootAttrPath) const InputAttrPath & lockRootAttrPath)
{ {
// Fetch a lazy tree first. // Fetch a lazy tree first.
auto cachedInput = auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries);
state.inputCache->getAccessor(state.fetchSettings, *state.store, originalRef.input, useRegistries);
auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir); auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir); auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
@@ -389,8 +388,7 @@ static Flake getFlake(
debug("refetching input '%s' due to self attribute", newLockedRef); debug("refetching input '%s' due to self attribute", newLockedRef);
// FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'. // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
newLockedRef.input.attrs.erase("narHash"); newLockedRef.input.attrs.erase("narHash");
auto cachedInput2 = state.inputCache->getAccessor( auto cachedInput2 = state.inputCache->getAccessor(state.store, newLockedRef.input, fetchers::UseRegistries::No);
state.fetchSettings, *state.store, newLockedRef.input, fetchers::UseRegistries::No);
cachedInput.accessor = cachedInput2.accessor; cachedInput.accessor = cachedInput2.accessor;
lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
} }
@@ -706,8 +704,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
this input. */ this input. */
debug("creating new input '%s'", inputAttrPathS); debug("creating new input '%s'", inputAttrPathS);
if (!lockFlags.allowUnlocked && !input.ref->input.isLocked(state.fetchSettings) if (!lockFlags.allowUnlocked && !input.ref->input.isLocked() && !input.ref->input.isRelative())
&& !input.ref->input.isRelative())
throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS); throw Error("cannot update unlocked flake input '%s' in pure mode", inputAttrPathS);
/* Note: in case of an --override-input, we use /* Note: in case of an --override-input, we use
@@ -756,7 +753,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
return {*resolvedPath, *input.ref}; return {*resolvedPath, *input.ref};
} else { } else {
auto cachedInput = state.inputCache->getAccessor( auto cachedInput = state.inputCache->getAccessor(
state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs); state.store, input.ref->input, useRegistriesInputs);
auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
@@ -956,7 +953,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
auto key = keyMap.find(node); auto key = keyMap.find(node);
assert(key != keyMap.end()); assert(key != keyMap.end());
override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel(), state.mem); override.alloc(state.symbols.create("dir")).mkString(CanonPath(subdir).rel());
overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); overrides.alloc(state.symbols.create(key->second)).mkAttrs(override);
} }
@@ -966,13 +963,16 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix")); Value * vCallFlake = requireInternalFile(state, CanonPath("call-flake.nix"));
auto vLocks = state.allocValue(); auto vLocks = state.allocValue();
vLocks->mkString(lockFileStr, state.mem); vLocks->mkString(lockFileStr);
Value * args[] = {vLocks, &vOverrides}; auto vFetchFinalTree = get(state.internalPrimOps, "fetchFinalTree");
assert(vFetchFinalTree);
Value * args[] = {vLocks, &vOverrides, *vFetchFinalTree};
state.callFunction(*vCallFlake, args, vRes, noPos); state.callFunction(*vCallFlake, args, vRes, noPos);
} }
std::optional<Fingerprint> LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const
{ {
if (lockFile.isUnlocked(fetchSettings)) if (lockFile.isUnlocked(fetchSettings))
return std::nullopt; return std::nullopt;
@@ -1002,7 +1002,7 @@ Flake::~Flake() {}
ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake) ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake)
{ {
auto fingerprint = state.settings.useEvalCache && state.settings.pureEval auto fingerprint = state.settings.useEvalCache && state.settings.pureEval
? lockedFlake->getFingerprint(*state.store, state.fetchSettings) ? lockedFlake->getFingerprint(state.store, state.fetchSettings)
: std::nullopt; : std::nullopt;
auto rootLoader = [&state, lockedFlake]() { auto rootLoader = [&state, lockedFlake]() {
/* For testing whether the evaluation cache is /* For testing whether the evaluation cache is
+4 -6
View File
@@ -64,10 +64,9 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef)
return str; return str;
} }
FlakeRef FlakeRef FlakeRef::resolve(ref<Store> store, fetchers::UseRegistries useRegistries) const
FlakeRef::resolve(const fetchers::Settings & fetchSettings, Store & store, fetchers::UseRegistries useRegistries) const
{ {
auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries); auto [input2, extraAttrs] = lookupInRegistries(store, input, useRegistries);
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
} }
@@ -288,10 +287,9 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Settings & fetchSettings, const fet
fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
} }
std::pair<ref<SourceAccessor>, FlakeRef> std::pair<ref<SourceAccessor>, FlakeRef> FlakeRef::lazyFetch(ref<Store> store) const
FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const
{ {
auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store); auto [accessor, lockedInput] = input.getAccessor(store);
return {accessor, FlakeRef(std::move(lockedInput), subdir)}; return {accessor, FlakeRef(std::move(lockedInput), subdir)};
} }
+1 -1
View File
@@ -135,7 +135,7 @@ struct LockedFlake
*/ */
std::map<ref<Node>, SourcePath> nodePaths; std::map<ref<Node>, SourcePath> nodePaths;
std::optional<Fingerprint> getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const; std::optional<Fingerprint> getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const;
}; };
struct LockFlags struct LockFlags
+2 -5
View File
@@ -71,14 +71,11 @@ struct FlakeRef
fetchers::Attrs toAttrs() const; fetchers::Attrs toAttrs() const;
FlakeRef resolve( FlakeRef resolve(ref<Store> store, fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
const fetchers::Settings & fetchSettings,
Store & store,
fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs); static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs);
std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const; std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(ref<Store> store) const;
/** /**
* Canonicalize a flakeref for the purpose of comparing "old" and * Canonicalize a flakeref for the purpose of comparing "old" and
+2 -2
View File
@@ -74,7 +74,7 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann:
, parentInputAttrPath( , parentInputAttrPath(
json.find("parent") != json.end() ? (std::optional<InputAttrPath>) json["parent"] : std::nullopt) json.find("parent") != json.end() ? (std::optional<InputAttrPath>) json["parent"] : std::nullopt)
{ {
if (!lockedRef.input.isLocked(fetchSettings) && !lockedRef.input.isRelative()) { if (!lockedRef.input.isLocked() && !lockedRef.input.isRelative()) {
if (lockedRef.input.getNarHash()) if (lockedRef.input.getNarHash())
warn( warn(
"Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. " "Lock file entry '%s' is unlocked (e.g. lacks a Git revision) but is checked by NAR hash. "
@@ -282,7 +282,7 @@ std::optional<FlakeRef> LockFile::isUnlocked(const fetchers::Settings & fetchSet
latter case, we can verify the input but we may not be able to latter case, we can verify the input but we may not be able to
fetch it from anywhere. */ fetch it from anywhere. */
auto isConsideredLocked = [&](const fetchers::Input & input) { auto isConsideredLocked = [&](const fetchers::Input & input) {
return input.isLocked(fetchSettings) || (fetchSettings.allowDirtyLocks && input.getNarHash()); return input.isLocked() || (fetchSettings.allowDirtyLocks && input.getNarHash());
}; };
for (auto & i : nodes) { for (auto & i : nodes) {
+5 -62
View File
@@ -467,6 +467,8 @@ public:
std::string getStatus(State & state) std::string getStatus(State & state)
{ {
auto MiB = 1024.0 * 1024.0;
std::string res; std::string res;
auto renderActivity = auto renderActivity =
@@ -514,65 +516,6 @@ public:
return s; return s;
}; };
auto renderSizeActivity = [&](ActivityType type, const std::string & itemFmt = "%s") {
auto & act = state.activitiesByType[type];
uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
for (auto & j : act.its) {
done += j.second->done;
expected += j.second->expected;
running += j.second->running;
failed += j.second->failed;
}
expected = std::max(expected, act.expected);
std::optional<SizeUnit> commonUnit;
std::string s;
if (running || done || expected || failed) {
if (running)
if (expected != 0) {
commonUnit = getCommonSizeUnit({(int64_t) running, (int64_t) done, (int64_t) expected});
s =
fmt(ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL "/%s",
commonUnit ? renderSizeWithoutUnit(running, *commonUnit) : renderSize(running),
commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done),
commonUnit ? renderSizeWithoutUnit(expected, *commonUnit) : renderSize(expected));
} else {
commonUnit = getCommonSizeUnit({(int64_t) running, (int64_t) done});
s =
fmt(ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL,
commonUnit ? renderSizeWithoutUnit(running, *commonUnit) : renderSize(running),
commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done));
}
else if (expected != done)
if (expected != 0) {
commonUnit = getCommonSizeUnit({(int64_t) done, (int64_t) expected});
s =
fmt(ANSI_GREEN "%s" ANSI_NORMAL "/%s",
commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done),
commonUnit ? renderSizeWithoutUnit(expected, *commonUnit) : renderSize(expected));
} else {
commonUnit = getSizeUnit(done);
s = fmt(ANSI_GREEN "%s" ANSI_NORMAL, renderSizeWithoutUnit(done, *commonUnit));
}
else {
commonUnit = getSizeUnit(done);
s = fmt(done ? ANSI_GREEN "%s" ANSI_NORMAL : "%s", renderSizeWithoutUnit(done, *commonUnit));
}
if (commonUnit)
s = fmt("%s %siB", s, getSizeUnitSuffix(*commonUnit));
s = fmt(itemFmt, s);
if (failed)
s += fmt(" (" ANSI_RED "%s failed" ANSI_NORMAL ")", renderSize(failed));
}
return s;
};
auto showActivity = auto showActivity =
[&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
auto s = renderActivity(type, itemFmt, numberFmt, unit); auto s = renderActivity(type, itemFmt, numberFmt, unit);
@@ -586,7 +529,7 @@ public:
showActivity(actBuilds, "%s built"); showActivity(actBuilds, "%s built");
auto s1 = renderActivity(actCopyPaths, "%s copied"); auto s1 = renderActivity(actCopyPaths, "%s copied");
auto s2 = renderSizeActivity(actCopyPath); auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB);
if (!s1.empty() || !s2.empty()) { if (!s1.empty() || !s2.empty()) {
if (!res.empty()) if (!res.empty())
@@ -602,12 +545,12 @@ public:
} }
} }
renderSizeActivity(actFileTransfer, "%s DL"); showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
{ {
auto s = renderActivity(actOptimiseStore, "%s paths optimised"); auto s = renderActivity(actOptimiseStore, "%s paths optimised");
if (s != "") { if (s != "") {
s += fmt(", %s / %d inodes freed", renderSize(state.bytesLinked), state.filesLinked); s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked);
if (!res.empty()) if (!res.empty())
res += ", "; res += ", ";
res += s; res += s;
-2
View File
@@ -35,8 +35,6 @@ include_dirs = [ include_directories('.') ]
headers = files( headers = files(
'nix_api_store.h', 'nix_api_store.h',
'nix_api_store/derivation.h',
'nix_api_store/store_path.h',
) )
# TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c`
+36 -2
View File
@@ -12,8 +12,6 @@
*/ */
#include "nix_api_util.h" #include "nix_api_util.h"
#include "nix_api_store/store_path.h"
#include "nix_api_store/derivation.h"
#include <stdbool.h> #include <stdbool.h>
#ifdef __cplusplus #ifdef __cplusplus
@@ -23,6 +21,10 @@ extern "C" {
/** @brief Reference to a Nix store */ /** @brief Reference to a Nix store */
typedef struct Store Store; typedef struct Store Store;
/** @brief Nix store path */
typedef struct StorePath StorePath;
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/** /**
* @brief Initializes the Nix store library * @brief Initializes the Nix store library
@@ -116,6 +118,30 @@ nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_ca
*/ */
StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
/**
* @brief Get the path name (e.g. "name" in /nix/store/...-name)
*
* @param[in] store_path the path to get the name from
* @param[in] callback called with the name
* @param[in] user_data arbitrary data, passed to the callback when it's called.
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
/**
* @brief Copy a StorePath
*
* @param[in] p the path to copy
* @return a new StorePath
*/
StorePath * nix_store_path_clone(const StorePath * p);
/** @brief Deallocate a StorePath
*
* Does not fail.
* @param[in] p the path to free
*/
void nix_store_path_free(StorePath * p);
/** /**
* @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in
* the store) * the store)
@@ -203,6 +229,14 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
*/ */
StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation); StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation);
/**
* @brief Deallocate a `nix_derivation`
*
* Does not fail.
* @param[in] drv the derivation to free
*/
void nix_derivation_free(nix_derivation * drv);
/** /**
* @brief Copy the closure of `path` from `srcStore` to `dstStore`. * @brief Copy the closure of `path` from `srcStore` to `dstStore`.
* *
-38
View File
@@ -1,38 +0,0 @@
#ifndef NIX_API_STORE_DERIVATION_H
#define NIX_API_STORE_DERIVATION_H
/**
* @defgroup libstore_derivation Derivation
* @ingroup libstore
* @brief Derivation operations that don't require a Store
* @{
*/
/** @file
* @brief Derivation operations
*/
#include "nix_api_util.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
/** @brief Nix Derivation */
typedef struct nix_derivation nix_derivation;
/**
* @brief Deallocate a `nix_derivation`
*
* Does not fail.
* @param[in] drv the derivation to free
*/
void nix_derivation_free(nix_derivation * drv);
// cffi end
#ifdef __cplusplus
}
#endif
/**
* @}
*/
#endif // NIX_API_STORE_DERIVATION_H
-54
View File
@@ -1,54 +0,0 @@
#ifndef NIX_API_STORE_STORE_PATH_H
#define NIX_API_STORE_STORE_PATH_H
/**
* @defgroup libstore_storepath StorePath
* @ingroup libstore
* @brief Store path operations that don't require a Store
* @{
*/
/** @file
* @brief Store path operations
*/
#include "nix_api_util.h"
#ifdef __cplusplus
extern "C" {
#endif
// cffi start
/** @brief Nix store path */
typedef struct StorePath StorePath;
/**
* @brief Copy a StorePath
*
* @param[in] p the path to copy
* @return a new StorePath
*/
StorePath * nix_store_path_clone(const StorePath * p);
/** @brief Deallocate a StorePath
*
* Does not fail.
* @param[in] p the path to free
*/
void nix_store_path_free(StorePath * p);
/**
* @brief Get the path name (e.g. "<name>" in /nix/store/<hash>-<name>)
*
* @param[in] store_path the path to get the name from
* @param[in] callback called with the name
* @param[in] user_data arbitrary data, passed to the callback when it's called.
*/
void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
// cffi end
#ifdef __cplusplus
}
#endif
/**
* @}
*/
#endif // NIX_API_STORE_STORE_PATH_H
Loaded 100 of 303 files, more files were not shown because too many files have changed in this diff. Show more