Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bb5d2e9676 | ||
|
|
8b0026312a |
No files matched your search
@@ -26,7 +26,7 @@ jobs:
|
||||
# required to find all branches
|
||||
fetch-depth: 0
|
||||
- name: Create backport PRs
|
||||
uses: korthout/backport-action@c656f5d5851037b2b38fb5db2691a03fa229e3b2 # v4.0.1
|
||||
uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
|
||||
id: backport
|
||||
with:
|
||||
# Config README: https://github.com/korthout/backport-action#backport-action
|
||||
|
||||
@@ -125,13 +125,13 @@ jobs:
|
||||
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
|
||||
if: ${{ matrix.instrumented }}
|
||||
- name: Upload coverage reports
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: coverage-reports
|
||||
path: coverage-reports/
|
||||
if: ${{ matrix.instrumented }}
|
||||
- name: Upload installer tarball
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: installer-${{matrix.os}}
|
||||
path: out/*
|
||||
@@ -164,7 +164,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Download installer tarball
|
||||
uses: actions/download-artifact@v7
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: installer-${{matrix.os}}
|
||||
path: out
|
||||
@@ -174,7 +174,7 @@ jobs:
|
||||
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
|
||||
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
|
||||
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
|
||||
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
|
||||
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
|
||||
if: ${{ !matrix.experimental-installer }}
|
||||
with:
|
||||
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
|
||||
|
||||
@@ -3,10 +3,6 @@
|
||||
This section provides some notes on how to start hacking on Nix.
|
||||
To get the latest version of Nix from GitHub:
|
||||
|
||||
> **Note**
|
||||
>
|
||||
> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo.
|
||||
|
||||
```console
|
||||
$ git clone https://github.com/NixOS/nix.git
|
||||
$ cd nix
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
{{#include build-trace-entry-v2-fixed.md}}
|
||||
{{#include build-trace-entry-v1-fixed.md}}
|
||||
|
||||
## Examples
|
||||
|
||||
### Simple build trace entry
|
||||
|
||||
```json
|
||||
{{#include schema/build-trace-entry-v2/simple.json}}
|
||||
{{#include schema/build-trace-entry-v1/simple.json}}
|
||||
```
|
||||
|
||||
### Build trace entry with dependencies
|
||||
|
||||
```json
|
||||
{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}}
|
||||
```
|
||||
|
||||
### Build trace entry with signature
|
||||
|
||||
```json
|
||||
{{#include schema/build-trace-entry-v2/with-signature.json}}
|
||||
{{#include schema/build-trace-entry-v1/with-signature.json}}
|
||||
```
|
||||
|
||||
<!--
|
||||
## Raw Schema
|
||||
|
||||
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json)
|
||||
-->
|
||||
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v1.json)
|
||||
-->
|
||||
@@ -17,7 +17,7 @@ schemas = [
|
||||
'derivation-v4',
|
||||
'derivation-options-v1',
|
||||
'deriving-path-v1',
|
||||
'build-trace-entry-v2',
|
||||
'build-trace-entry-v1',
|
||||
'build-result-v1',
|
||||
'store-v1',
|
||||
]
|
||||
|
||||
@@ -83,7 +83,7 @@ properties:
|
||||
description: |
|
||||
A mapping from output names to their build trace entries.
|
||||
additionalProperties:
|
||||
"$ref": "build-trace-entry-v2.yaml"
|
||||
"$ref": "build-trace-entry-v1.yaml"
|
||||
|
||||
failure:
|
||||
type: object
|
||||
|
||||
File renamed without changes.
+18
-13
@@ -1,5 +1,5 @@
|
||||
"$schema": "http://json-schema.org/draft-04/schema"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json"
|
||||
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json"
|
||||
title: Build Trace Entry
|
||||
description: |
|
||||
A record of a successful build outcome for a specific derivation output.
|
||||
@@ -11,17 +11,10 @@ description: |
|
||||
> This JSON format is currently
|
||||
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
|
||||
> and subject to change.
|
||||
|
||||
Verision history:
|
||||
|
||||
- Version 1: Original format
|
||||
|
||||
- Version 2: Remove `dependentRealisations`
|
||||
|
||||
type: object
|
||||
required:
|
||||
- id
|
||||
- outPath
|
||||
- dependentRealisations
|
||||
- signatures
|
||||
allOf:
|
||||
- "$ref": "#/$defs/key"
|
||||
@@ -29,11 +22,9 @@ allOf:
|
||||
properties:
|
||||
id: {}
|
||||
outPath: {}
|
||||
dependentRealisations: {}
|
||||
signatures: {}
|
||||
additionalProperties:
|
||||
dependentRealisations:
|
||||
description: deprecated field
|
||||
type: object
|
||||
additionalProperties: false
|
||||
|
||||
"$defs":
|
||||
key:
|
||||
@@ -69,6 +60,7 @@ additionalProperties:
|
||||
type: object
|
||||
required:
|
||||
- outPath
|
||||
- dependentRealisations
|
||||
- signatures
|
||||
properties:
|
||||
outPath:
|
||||
@@ -77,6 +69,19 @@ additionalProperties:
|
||||
description: |
|
||||
The path to the store object that resulted from building this derivation for the given output name.
|
||||
|
||||
dependentRealisations:
|
||||
type: object
|
||||
title: Underlying Base Build Trace
|
||||
description: |
|
||||
This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
|
||||
|
||||
Keys are derivation output IDs (same format as the main `id` field).
|
||||
Values are the store paths that those dependencies resolved to.
|
||||
|
||||
As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
|
||||
This is the set of base build trace entries that this derived build trace is derived from.
|
||||
(The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
|
||||
|
||||
patternProperties:
|
||||
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
|
||||
"$ref": "store-path-v1.yaml"
|
||||
@@ -70,7 +70,7 @@ properties:
|
||||
"^[A-Za-z0-9+/]{43}=$":
|
||||
type: object
|
||||
additionalProperties:
|
||||
"$ref": "./build-trace-entry-v2.yaml#/$defs/value"
|
||||
"$ref": "./build-trace-entry-v1.yaml#/$defs/value"
|
||||
additionalProperties: false
|
||||
|
||||
"$defs":
|
||||
|
||||
@@ -62,11 +62,9 @@ schemas = [
|
||||
},
|
||||
{
|
||||
'stem' : 'build-trace-entry',
|
||||
'schema' : schema_dir / 'build-trace-entry-v2.yaml',
|
||||
'schema' : schema_dir / 'build-trace-entry-v1.yaml',
|
||||
'files' : [
|
||||
'simple.json',
|
||||
# The field is no longer supported, but we want to show that we
|
||||
# ignore it during parsing.
|
||||
'with-dependent-realisations.json',
|
||||
'with-signature.json',
|
||||
],
|
||||
|
||||
@@ -741,11 +741,6 @@ public:
|
||||
inDebugger = true;
|
||||
}
|
||||
|
||||
DebuggerGuard(DebuggerGuard &&) = delete;
|
||||
DebuggerGuard(const DebuggerGuard &) = delete;
|
||||
DebuggerGuard & operator=(DebuggerGuard &&) = delete;
|
||||
DebuggerGuard & operator=(const DebuggerGuard &) = delete;
|
||||
|
||||
~DebuggerGuard()
|
||||
{
|
||||
inDebugger = false;
|
||||
|
||||
@@ -107,8 +107,6 @@ private:
|
||||
Bindings & operator=(const Bindings &) = delete;
|
||||
Bindings & operator=(Bindings &&) = delete;
|
||||
|
||||
~Bindings() = default;
|
||||
|
||||
friend class BindingsBuilder;
|
||||
|
||||
/**
|
||||
|
||||
@@ -164,6 +164,8 @@ public:
|
||||
Value ** elems;
|
||||
ListBuilder(EvalMemory & mem, size_t size);
|
||||
|
||||
// NOTE: Can be noexcept because we are just copying integral values and
|
||||
// raw pointers.
|
||||
ListBuilder(ListBuilder && x) noexcept
|
||||
: size(x.size)
|
||||
, inlineElems{x.inlineElems[0], x.inlineElems[1]}
|
||||
@@ -171,11 +173,6 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
ListBuilder(const ListBuilder &) = delete;
|
||||
ListBuilder & operator=(ListBuilder &&) = delete;
|
||||
ListBuilder & operator=(const ListBuilder &) = delete;
|
||||
~ListBuilder() = default;
|
||||
|
||||
Value *& operator[](size_t n)
|
||||
{
|
||||
return elems[n];
|
||||
|
||||
+2
-18
@@ -4061,8 +4061,6 @@ static RegisterPrimOp primop_sort({
|
||||
|
||||
1. Transitivity
|
||||
|
||||
If a is less than b and b is less than c, then it follows that a is less than c.
|
||||
|
||||
```nix
|
||||
comparator a b && comparator b c -> comparator a c
|
||||
```
|
||||
@@ -4075,23 +4073,9 @@ static RegisterPrimOp primop_sort({
|
||||
|
||||
1. Transitivity of equivalence
|
||||
|
||||
First, two values a and b are considered equivalent with respect to the comparator if:
|
||||
|
||||
```
|
||||
!comparator a b && !comparator b a
|
||||
```
|
||||
|
||||
In other words, neither is considered "less than" the other.
|
||||
|
||||
Transitivity of equivalence means:
|
||||
|
||||
If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c.
|
||||
|
||||
```nix
|
||||
let
|
||||
equiv = x: y: (!comparator x y && !comparator y x);
|
||||
in
|
||||
equiv a b && equiv b c -> equiv a c
|
||||
let equiv = a: b: (!comparator a b && !comparator b a); in
|
||||
equiv a b && equiv b c -> equiv a c
|
||||
```
|
||||
|
||||
If the *comparator* violates any of these properties, then `builtins.sort`
|
||||
|
||||
@@ -70,6 +70,12 @@ TEST_F(GitUtilsTest, sink_basic)
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
// TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory
|
||||
// here
|
||||
// The sync method does not document this behavior, should probably renamed because it's not very
|
||||
// general, and I can't imagine that "non-conventional" archives or any other source to be handled by
|
||||
// this sink.
|
||||
|
||||
sink->createDirectory(CanonPath("foo-1.1"));
|
||||
|
||||
sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) {
|
||||
@@ -85,7 +91,7 @@ TEST_F(GitUtilsTest, sink_basic)
|
||||
|
||||
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
|
||||
|
||||
auto result = repo->dereferenceSingletonDirectory(sink->flush().hash);
|
||||
auto result = repo->dereferenceSingletonDirectory(sink->flush());
|
||||
auto accessor = repo->getAccessor(result, {}, getRepoName());
|
||||
auto entries = accessor->readDirectory(CanonPath::root);
|
||||
ASSERT_EQ(entries.size(), 5u);
|
||||
@@ -117,51 +123,6 @@ TEST_F(GitUtilsTest, sink_hardlink)
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, sink_root_file)
|
||||
{
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
sink->createRegularFile(
|
||||
CanonPath::root, [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
|
||||
|
||||
auto result = sink->flush();
|
||||
ASSERT_EQ(result.mode, nix::git::Mode::Regular);
|
||||
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
|
||||
ASSERT_EQ(accessor->readFile(CanonPath::root), "hello world");
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, sink_root_symlink)
|
||||
{
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
sink->createSymlink(CanonPath::root, "target");
|
||||
|
||||
auto result = sink->flush();
|
||||
ASSERT_EQ(result.mode, nix::git::Mode::Symlink);
|
||||
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
|
||||
ASSERT_EQ(accessor->readLink(CanonPath::root), "target");
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, sink_hardlink_to_root)
|
||||
{
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
sink->createRegularFile(
|
||||
CanonPath("hello"), [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
|
||||
|
||||
sink->createDirectory(CanonPath("subdir"));
|
||||
sink->createHardlink(CanonPath("subdir/link"), CanonPath("hello"));
|
||||
|
||||
auto result = sink->flush();
|
||||
ASSERT_EQ(result.mode, nix::git::Mode::Directory);
|
||||
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
|
||||
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
|
||||
ASSERT_EQ(accessor->readFile(CanonPath("subdir/link")), "hello world");
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, peel_reference)
|
||||
{
|
||||
// Create a commit in the repo
|
||||
|
||||
+99
-192
@@ -962,15 +962,15 @@ struct GitSourceAccessor : SourceAccessor
|
||||
|
||||
Blob getBlob(State & state, const CanonPath & path, bool expectSymlink)
|
||||
{
|
||||
if (!expectSymlink && git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
|
||||
return dupObject<Blob>((git_blob *) &*state.root);
|
||||
|
||||
auto notExpected = [&]() {
|
||||
throw Error(expectSymlink ? "'%s' is not a symlink" : "'%s' is not a regular file", showPath(path));
|
||||
};
|
||||
|
||||
if (path.isRoot()) {
|
||||
if (git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
|
||||
return dupObject<Blob>((git_blob *) &*state.root);
|
||||
if (path.isRoot())
|
||||
notExpected();
|
||||
}
|
||||
|
||||
auto entry = need(state, path);
|
||||
|
||||
@@ -1054,24 +1054,16 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
|
||||
}
|
||||
};
|
||||
|
||||
namespace {
|
||||
|
||||
struct DirEntry
|
||||
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
{
|
||||
git_oid oid;
|
||||
git_filemode_t mode;
|
||||
};
|
||||
ref<GitRepoImpl> repo;
|
||||
|
||||
struct DirectoryState
|
||||
{
|
||||
struct PendingDir
|
||||
{
|
||||
std::string name;
|
||||
TreeBuilder builder;
|
||||
};
|
||||
|
||||
GitRepoImpl & repo;
|
||||
TreeBuilder rootBuilder;
|
||||
std::vector<PendingDir> pendingDirs;
|
||||
|
||||
/**
|
||||
@@ -1086,9 +1078,30 @@ struct DirectoryState
|
||||
*/
|
||||
decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
|
||||
|
||||
DirectoryState(GitRepoImpl & repo)
|
||||
void pushBuilder(std::string name)
|
||||
{
|
||||
const git_tree_entry * entry;
|
||||
Tree prevTree = nullptr;
|
||||
|
||||
if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
|
||||
/* Clone a tree that we've already finished. This happens
|
||||
if a tarball has directory entries that are not
|
||||
contiguous. */
|
||||
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
|
||||
throw Error("parent of '%s' is not a directory", name);
|
||||
|
||||
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
|
||||
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
|
||||
}
|
||||
|
||||
git_treebuilder * b;
|
||||
if (git_treebuilder_new(&b, *repo, prevTree.get()))
|
||||
throw Error("creating a tree builder: %s", git_error_last()->message);
|
||||
pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
|
||||
};
|
||||
|
||||
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
|
||||
: repo(repo)
|
||||
, rootBuilder(makeBuilder())
|
||||
{
|
||||
/* Monkey-patching the pack backend to only read the pack directory
|
||||
once. Otherwise it will do a readdir for each added oid when it's
|
||||
@@ -1113,145 +1126,62 @@ struct DirectoryState
|
||||
*/
|
||||
packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
|
||||
}
|
||||
pushBuilder("");
|
||||
}
|
||||
|
||||
TreeBuilder makeBuilder(git_tree * prevTree = nullptr);
|
||||
|
||||
std::vector<std::string> prepareDirs(const CanonPath & path);
|
||||
void updateBuilders(std::span<const std::string> names);
|
||||
std::pair<git_oid, std::string> popBuilder();
|
||||
void pushBuilder(std::string name);
|
||||
void addToTree(const std::string & name, const DirEntry & entry);
|
||||
};
|
||||
|
||||
TreeBuilder DirectoryState::makeBuilder(git_tree * prevTree)
|
||||
{
|
||||
TreeBuilder builder;
|
||||
if (git_treebuilder_new(Setter(builder), repo, prevTree))
|
||||
throw Error("creating a tree builder: %s", git_error_last()->message);
|
||||
return builder;
|
||||
}
|
||||
|
||||
git_oid finishBuilder(TreeBuilder & builder)
|
||||
{
|
||||
git_oid oid;
|
||||
if (git_treebuilder_write(&oid, builder.get()))
|
||||
throw Error("creating a tree object: %s", git_error_last()->message);
|
||||
return oid;
|
||||
}
|
||||
|
||||
void DirectoryState::pushBuilder(std::string name)
|
||||
{
|
||||
auto & parentBuilder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
|
||||
const git_tree_entry * entry;
|
||||
Tree prevTree = nullptr;
|
||||
|
||||
if ((entry = git_treebuilder_get(parentBuilder.get(), name.c_str()))) {
|
||||
/* Clone a tree that we've already finished. This happens
|
||||
if a tarball has directory entries that are not
|
||||
contiguous. */
|
||||
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
|
||||
throw Error("parent of '%s' is not a directory", name);
|
||||
|
||||
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), repo, entry))
|
||||
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
|
||||
}
|
||||
|
||||
pendingDirs.push_back({
|
||||
.name = std::move(name),
|
||||
.builder = makeBuilder(prevTree.get()),
|
||||
});
|
||||
}
|
||||
|
||||
std::pair<git_oid, std::string> DirectoryState::popBuilder()
|
||||
{
|
||||
assert(!pendingDirs.empty());
|
||||
auto pending = std::move(pendingDirs.back());
|
||||
pendingDirs.pop_back();
|
||||
return {finishBuilder(pending.builder), pending.name};
|
||||
}
|
||||
|
||||
void DirectoryState::addToTree(const std::string & name, const DirEntry & entry)
|
||||
{
|
||||
auto & builder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
|
||||
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &entry.oid, entry.mode))
|
||||
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
|
||||
}
|
||||
|
||||
void DirectoryState::updateBuilders(std::span<const std::string> names)
|
||||
{
|
||||
// Find the common prefix of pendingDirs and names.
|
||||
size_t prefixLen = 0;
|
||||
for (; prefixLen < names.size() && prefixLen < pendingDirs.size(); ++prefixLen)
|
||||
if (names[prefixLen] != pendingDirs[prefixLen].name)
|
||||
break;
|
||||
|
||||
// Finish the builders that are not part of the common prefix.
|
||||
for (auto n = pendingDirs.size(); n > prefixLen; --n) {
|
||||
auto [oid, name] = popBuilder();
|
||||
addToTree(name, DirEntry{.oid = oid, .mode = GIT_FILEMODE_TREE});
|
||||
}
|
||||
|
||||
// Create builders for the new directories.
|
||||
for (auto n = prefixLen; n < names.size(); ++n)
|
||||
pushBuilder(names[n]);
|
||||
}
|
||||
|
||||
std::vector<std::string> DirectoryState::prepareDirs(const CanonPath & path)
|
||||
{
|
||||
std::vector<std::string> pathComponents;
|
||||
for (auto & c : path)
|
||||
pathComponents.emplace_back(c);
|
||||
|
||||
updateBuilders(pathComponents);
|
||||
|
||||
return pathComponents;
|
||||
}
|
||||
|
||||
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
{
|
||||
ref<GitRepoImpl> repo;
|
||||
|
||||
/**
|
||||
* Root state:
|
||||
* - nullopt: nothing created yet
|
||||
* - DirEntry: root is a single non-directory object (file/symlink)
|
||||
* - DirectoryState: root is a directory with builders
|
||||
*/
|
||||
std::optional<std::variant<DirEntry, DirectoryState>> root;
|
||||
|
||||
DirectoryState & ensureDirectoryState()
|
||||
std::pair<git_oid, std::string> popBuilder()
|
||||
{
|
||||
if (!root) {
|
||||
root.emplace<DirectoryState>(*repo);
|
||||
}
|
||||
if (auto * dirState = std::get_if<DirectoryState>(&*root)) {
|
||||
return *dirState;
|
||||
}
|
||||
throw Error("cannot create directory entry: root is not a directory");
|
||||
}
|
||||
assert(!pendingDirs.empty());
|
||||
auto pending = std::move(pendingDirs.back());
|
||||
git_oid oid;
|
||||
if (git_treebuilder_write(&oid, pending.builder.get()))
|
||||
throw Error("creating a tree object: %s", git_error_last()->message);
|
||||
pendingDirs.pop_back();
|
||||
return {oid, pending.name};
|
||||
};
|
||||
|
||||
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
|
||||
: repo(repo)
|
||||
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
|
||||
{
|
||||
}
|
||||
assert(!pendingDirs.empty());
|
||||
auto & pending = pendingDirs.back();
|
||||
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
|
||||
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
|
||||
};
|
||||
|
||||
void addEntry(const CanonPath & path, const DirEntry & entry)
|
||||
void updateBuilders(std::span<const std::string> names)
|
||||
{
|
||||
if (auto res = path.parentAndChild()) {
|
||||
auto & [parent, child] = *res;
|
||||
auto & dirState = ensureDirectoryState();
|
||||
dirState.prepareDirs(parent);
|
||||
dirState.addToTree(std::string(child), entry);
|
||||
} else {
|
||||
if (root)
|
||||
throw Error("root already exists");
|
||||
root = entry;
|
||||
// Find the common prefix of pendingDirs and names.
|
||||
size_t prefixLen = 0;
|
||||
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
|
||||
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
|
||||
break;
|
||||
|
||||
// Finish the builders that are not part of the common prefix.
|
||||
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
|
||||
auto [oid, name] = popBuilder();
|
||||
addToTree(name, oid, GIT_FILEMODE_TREE);
|
||||
}
|
||||
|
||||
// Create builders for the new directories.
|
||||
for (auto n = prefixLen; n < names.size(); ++n)
|
||||
pushBuilder(names[n]);
|
||||
};
|
||||
|
||||
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
|
||||
{
|
||||
std::span<const std::string> pathComponents2{pathComponents};
|
||||
|
||||
updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
|
||||
if (stream)
|
||||
stream->free(stream);
|
||||
@@ -1330,68 +1260,60 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
"creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
|
||||
}
|
||||
|
||||
addEntry(
|
||||
path,
|
||||
DirEntry{
|
||||
.oid = oid,
|
||||
.mode = crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB,
|
||||
});
|
||||
addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
ensureDirectoryState().prepareDirs(path);
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
(void) prepareDirs(pathComponents, true);
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
git_oid oid;
|
||||
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
|
||||
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
|
||||
|
||||
addEntry(
|
||||
path,
|
||||
DirEntry{
|
||||
.oid = oid,
|
||||
.mode = GIT_FILEMODE_LINK,
|
||||
});
|
||||
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
|
||||
}
|
||||
|
||||
void createHardlink(const CanonPath & path, const CanonPath & target) override
|
||||
{
|
||||
auto & dirState = ensureDirectoryState();
|
||||
std::vector<std::string> pathComponents;
|
||||
for (auto & c : path)
|
||||
pathComponents.emplace_back(c);
|
||||
|
||||
auto res = path.parentAndChild();
|
||||
if (!res)
|
||||
throw Error("hard link cannot be root file system object");
|
||||
auto & [parent, name] = *res;
|
||||
|
||||
dirState.prepareDirs(parent);
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
// We can't just look up the path from the start of the root, since
|
||||
// some parent directories may not have finished yet, so we compute
|
||||
// a relative path that helps us find the right git_tree_builder or object.
|
||||
auto relTarget = parent.makeRelative(target);
|
||||
auto relTarget = CanonPath(path).parent()->makeRelative(target);
|
||||
|
||||
auto dir = dirState.pendingDirs.rbegin();
|
||||
auto dir = pendingDirs.rbegin();
|
||||
|
||||
// For each ../ component at the start, go up one directory.
|
||||
// CanonPath::makeRelative() always puts all .. elements at the start,
|
||||
// so they're all handled by this loop:
|
||||
std::string_view relTargetLeft(relTarget);
|
||||
while (hasPrefix(relTargetLeft, "../")) {
|
||||
if (dir == dirState.pendingDirs.rend())
|
||||
if (dir == pendingDirs.rend())
|
||||
throw Error("invalid hard link target '%s' for path '%s'", target, path);
|
||||
++dir;
|
||||
relTargetLeft = relTargetLeft.substr(3);
|
||||
}
|
||||
if (dir == pendingDirs.rend())
|
||||
throw Error("invalid hard link target '%s' for path '%s'", target, path);
|
||||
|
||||
// Look up the remainder of the target, starting at the
|
||||
// top-most `git_treebuilder`.
|
||||
std::variant<git_treebuilder *, git_oid> curDir{
|
||||
dir == dirState.pendingDirs.rend() ? dirState.rootBuilder.get() : dir->builder.get()};
|
||||
std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
|
||||
Object tree; // needed to keep `entry` alive
|
||||
const git_tree_entry * entry = nullptr;
|
||||
|
||||
@@ -1411,41 +1333,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
|
||||
assert(entry);
|
||||
|
||||
dirState.addToTree(
|
||||
std::string(name),
|
||||
DirEntry{.oid = *git_tree_entry_id(entry), .mode = git_tree_entry_filemode(entry)});
|
||||
addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
|
||||
}
|
||||
|
||||
git::TreeEntry flush() override
|
||||
Hash flush() override
|
||||
{
|
||||
if (!root)
|
||||
throw Error("nothing to flush");
|
||||
updateBuilders({});
|
||||
|
||||
auto [oid, mode] = std::visit(
|
||||
overloaded{
|
||||
[](DirEntry & entry) -> std::pair<git_oid, git_filemode_t> {
|
||||
return {entry.oid, entry.mode};
|
||||
},
|
||||
[](DirectoryState & dirState) -> std::pair<git_oid, git_filemode_t> {
|
||||
dirState.updateBuilders({});
|
||||
return {finishBuilder(dirState.rootBuilder), GIT_FILEMODE_TREE};
|
||||
},
|
||||
},
|
||||
*root);
|
||||
auto [oid, _name] = popBuilder();
|
||||
|
||||
if (auto * backend = repo->packBackend) {
|
||||
/* We are done writing blobs, can restore refresh functionality. */
|
||||
backend->refresh = dirState.packfileOdbRefresh;
|
||||
backend->refresh = packfileOdbRefresh;
|
||||
}
|
||||
|
||||
repo->flush();
|
||||
|
||||
return git::TreeEntry{.mode = static_cast<git::Mode>(mode), .hash = toHash(oid)};
|
||||
return toHash(oid);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
|
||||
{
|
||||
auto self = ref<GitRepoImpl>(shared_from_this());
|
||||
|
||||
@@ -321,12 +321,12 @@ struct GitArchiveInputScheme : InputScheme
|
||||
auto tarballCache = settings.getTarballCache();
|
||||
auto parseSink = tarballCache->getFileSystemObjectSink();
|
||||
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
||||
auto treeEntry = parseSink->flush();
|
||||
auto tree = parseSink->flush();
|
||||
|
||||
act.reset();
|
||||
|
||||
TarballInfo tarballInfo{
|
||||
.treeHash = tarballCache->dereferenceSingletonDirectory(treeEntry.hash), .lastModified = lastModified};
|
||||
.treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified};
|
||||
|
||||
cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
|
||||
cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
#include "nix/fetchers/filtering-source-accessor.hh"
|
||||
#include "nix/util/fs-sink.hh"
|
||||
#include "nix/util/git.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -18,9 +17,9 @@ struct Settings;
|
||||
struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
|
||||
{
|
||||
/**
|
||||
* Flush builder and return the final Git tree entry (hash and mode).
|
||||
* Flush builder and return a final Git hash.
|
||||
*/
|
||||
virtual git::TreeEntry flush() = 0;
|
||||
virtual Hash flush() = 0;
|
||||
};
|
||||
|
||||
struct GitAccessorOptions
|
||||
|
||||
@@ -182,7 +182,7 @@ static DownloadTarballResult downloadTarball_(
|
||||
auto tarballCache = settings.getTarballCache();
|
||||
auto parseSink = tarballCache->getFileSystemObjectSink();
|
||||
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
||||
auto treeEntry = parseSink->flush();
|
||||
auto tree = parseSink->flush();
|
||||
|
||||
act.reset();
|
||||
|
||||
@@ -196,7 +196,7 @@ static DownloadTarballResult downloadTarball_(
|
||||
infoAttrs = cached->value;
|
||||
} else {
|
||||
infoAttrs.insert_or_assign("etag", res->etag);
|
||||
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(treeEntry.hash).gitRev());
|
||||
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(tree).gitRev());
|
||||
infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
|
||||
if (res->immutableUrl)
|
||||
infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);
|
||||
|
||||
@@ -88,38 +88,25 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VERSION, VALUE); \
|
||||
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VERSION, VALUE); \
|
||||
} \
|
||||
TEST_F(FIXTURE, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VERSION, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VERSION, VALUE); \
|
||||
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(FIXTURE, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
|
||||
|
||||
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
TEST_F(FIXTURE, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
|
||||
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
|
||||
|
||||
} // namespace nix
|
||||
@@ -47,30 +47,24 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(CommonProtoTest, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(CommonProtoTest, NAME##_read) \
|
||||
{ \
|
||||
readProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_read) \
|
||||
{ \
|
||||
readJsonTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
TEST_F(CommonProtoTest, NAME##_write) \
|
||||
{ \
|
||||
writeProtoTest(STEM, VALUE); \
|
||||
} \
|
||||
TEST_F(CommonProtoTest, NAME##_json_write) \
|
||||
{ \
|
||||
writeJsonTest(STEM, VALUE); \
|
||||
}
|
||||
|
||||
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
|
||||
WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
|
||||
|
||||
CHARACTERIZATION_TEST(
|
||||
string,
|
||||
"string",
|
||||
@@ -147,7 +141,7 @@ CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
READ_CHARACTERIZATION_TEST(
|
||||
CHARACTERIZATION_TEST(
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
(std::tuple<Realisation>{
|
||||
@@ -155,6 +149,16 @@ READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
.dependentRealisations =
|
||||
{
|
||||
{
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
@@ -1,58 +0,0 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the output of a CA derivation",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"narSize": 152,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "test-ca-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {},
|
||||
"derivations": {
|
||||
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "test-ca-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the output of a CA derivation",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
|
||||
"narSize": 152,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"narSize": 232,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": [
|
||||
"out"
|
||||
]
|
||||
}
|
||||
},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "root-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
},
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "dep-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {},
|
||||
"derivations": {
|
||||
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"dynamicOutputs": {},
|
||||
"outputs": [
|
||||
"out"
|
||||
]
|
||||
}
|
||||
},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "root-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
},
|
||||
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
|
||||
"args": [],
|
||||
"builder": "",
|
||||
"env": {},
|
||||
"inputs": {
|
||||
"drvs": {},
|
||||
"srcs": []
|
||||
},
|
||||
"name": "dep-drv",
|
||||
"outputs": {
|
||||
"out": {
|
||||
"hashAlgo": "sha256",
|
||||
"method": "nar"
|
||||
}
|
||||
},
|
||||
"system": "",
|
||||
"version": 4
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
{
|
||||
"buildTrace": {
|
||||
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
},
|
||||
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
|
||||
"out": {
|
||||
"dependentRealisations": {},
|
||||
"outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
|
||||
"signatures": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
|
||||
"narSize": 232,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
},
|
||||
"w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": {
|
||||
"contents": {
|
||||
"contents": "I am the dependency output",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
|
||||
"narSize": 144,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": {
|
||||
"contents": {
|
||||
"contents": "Hello, world!",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
|
||||
"narSize": 128,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
|
||||
"contents": {
|
||||
"contents": "I am a dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"narSize": 136,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
},
|
||||
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
|
||||
"contents": {
|
||||
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"narSize": 184,
|
||||
"references": [
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
|
||||
],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
{
|
||||
"buildTrace": {},
|
||||
"config": {
|
||||
"store": "/nix/store"
|
||||
},
|
||||
"contents": {
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
|
||||
"contents": {
|
||||
"contents": "I am a dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
|
||||
"narSize": 136,
|
||||
"references": [],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
},
|
||||
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
|
||||
"contents": {
|
||||
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
|
||||
"executable": false,
|
||||
"type": "regular"
|
||||
},
|
||||
"info": {
|
||||
"ca": {
|
||||
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"method": "nar"
|
||||
},
|
||||
"deriver": null,
|
||||
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
|
||||
"narSize": 184,
|
||||
"references": [
|
||||
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
|
||||
],
|
||||
"registrationTime": null,
|
||||
"signatures": [],
|
||||
"storeDir": "/nix/store",
|
||||
"ultimate": false,
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"derivations": {}
|
||||
}
|
||||
@@ -85,7 +85,6 @@ sources = files(
|
||||
'store-reference.cc',
|
||||
'uds-remote-store.cc',
|
||||
'worker-protocol.cc',
|
||||
'worker-substitution.cc',
|
||||
'write-derivation.cc',
|
||||
)
|
||||
|
||||
|
||||
@@ -44,45 +44,54 @@ TEST_P(RealisationJsonTest, to_json)
|
||||
writeJsonTest(name, value);
|
||||
}
|
||||
|
||||
Realisation simple{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
RealisationJSON,
|
||||
RealisationJsonTest,
|
||||
::testing::Values(
|
||||
std::pair{
|
||||
"simple",
|
||||
simple,
|
||||
},
|
||||
std::pair{
|
||||
"with-signature",
|
||||
[&] {
|
||||
auto r = simple;
|
||||
// FIXME actually sign properly
|
||||
r.signatures = {"asdfasdfasdf"};
|
||||
return r;
|
||||
}(),
|
||||
}));
|
||||
([] {
|
||||
Realisation simple{
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
};
|
||||
return ::testing::Values(
|
||||
std::pair{
|
||||
"simple",
|
||||
simple,
|
||||
},
|
||||
std::pair{
|
||||
"with-signature",
|
||||
[&] {
|
||||
auto r = simple;
|
||||
// FIXME actually sign properly
|
||||
r.signatures = {"asdfasdfasdf"};
|
||||
return r;
|
||||
}()},
|
||||
std::pair{
|
||||
"with-dependent-realisations",
|
||||
[&] {
|
||||
auto r = simple;
|
||||
r.dependentRealisations = {{
|
||||
{
|
||||
.drvHash = Hash::parseExplicitFormatUnprefixed(
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
|
||||
HashAlgorithm::SHA256,
|
||||
HashFormat::Base16),
|
||||
.outputName = "foo",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
|
||||
}};
|
||||
return r;
|
||||
}(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* We no longer have a notion of "dependent realisations", but we still
|
||||
* want to parse old realisation files. So make this just be a read test
|
||||
* (no write direction), accordingly.
|
||||
*/
|
||||
TEST_F(RealisationTest, dependent_realisations_from_json)
|
||||
{
|
||||
readJsonTest("with-dependent-realisations", simple);
|
||||
}
|
||||
()));
|
||||
|
||||
} // namespace nix
|
||||
@@ -118,7 +118,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
ServeProtoTest,
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
@@ -128,6 +128,16 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
.dependentRealisations =
|
||||
{
|
||||
{
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
@@ -171,7 +171,7 @@ VERSIONED_CHARACTERIZATION_TEST(
|
||||
},
|
||||
}))
|
||||
|
||||
VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
VERSIONED_CHARACTERIZATION_TEST(
|
||||
WorkerProtoTest,
|
||||
realisation_with_deps,
|
||||
"realisation-with-deps",
|
||||
@@ -181,6 +181,16 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
|
||||
{
|
||||
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
.signatures = {"asdf", "qwer"},
|
||||
.dependentRealisations =
|
||||
{
|
||||
{
|
||||
DrvOutput{
|
||||
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
|
||||
.outputName = "quux",
|
||||
},
|
||||
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
|
||||
|
||||
@@ -1,450 +0,0 @@
|
||||
#include <gtest/gtest.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/derivations.hh"
|
||||
#include "nix/store/dummy-store-impl.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
#include "nix/util/memory-source-accessor.hh"
|
||||
|
||||
#include "nix/store/tests/libstore.hh"
|
||||
#include "nix/util/tests/json-characterization.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest<ref<DummyStore>>
|
||||
{
|
||||
std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution";
|
||||
|
||||
protected:
|
||||
ref<DummyStore> dummyStore;
|
||||
ref<DummyStore> substituter;
|
||||
|
||||
WorkerSubstitutionTest()
|
||||
: LibStoreTest([] {
|
||||
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
|
||||
config->readOnly = false;
|
||||
return config->openDummyStore();
|
||||
}())
|
||||
, dummyStore(store.dynamic_pointer_cast<DummyStore>())
|
||||
, substituter([] {
|
||||
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
|
||||
config->readOnly = false;
|
||||
config->isTrusted = true;
|
||||
return config->openDummyStore();
|
||||
}())
|
||||
{
|
||||
}
|
||||
|
||||
public:
|
||||
std::filesystem::path goldenMaster(std::string_view testStem) const override
|
||||
{
|
||||
return unitTestData / testStem;
|
||||
}
|
||||
|
||||
static void SetUpTestSuite()
|
||||
{
|
||||
initLibStore(false);
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(WorkerSubstitutionTest, singleStoreObject)
|
||||
{
|
||||
// Add a store path to the substituter
|
||||
auto pathInSubstituter = substituter->addToStore(
|
||||
"hello",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "Hello, world!",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Snapshot the substituter (has one store object)
|
||||
checkpointJson("single/substituter", substituter);
|
||||
|
||||
// Snapshot the destination store before (should be empty)
|
||||
checkpointJson("../dummy-store/empty", dummyStore);
|
||||
|
||||
// The path should not exist in the destination store yet
|
||||
ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituerAsStore = substituter;
|
||||
worker.getSubstituters = [substituerAsStore]() -> std::list<ref<Store>> { return {substituerAsStore}; };
|
||||
|
||||
// Create a substitution goal for the path
|
||||
auto goal = worker.makePathSubstitutionGoal(pathInSubstituter);
|
||||
|
||||
// Run the worker with -j0 semantics (no local builds, only substitution)
|
||||
// The worker.run() takes a set of goals
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after (should match the substituter)
|
||||
checkpointJson("single/substituter", dummyStore);
|
||||
|
||||
// The path should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter));
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
}
|
||||
|
||||
TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject)
|
||||
{
|
||||
// First, add a dependency store path to the substituter
|
||||
auto dependencyPath = substituter->addToStore(
|
||||
"dependency",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I am a dependency",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Now add a store path that references the dependency
|
||||
auto mainPath = substituter->addToStore(
|
||||
"main",
|
||||
SourcePath{
|
||||
[&] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
// Include a reference to the dependency path in the contents
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I depend on " + substituter->printStorePath(dependencyPath),
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256,
|
||||
StorePathSet{dependencyPath});
|
||||
|
||||
// Snapshot the substituter (has two store objects)
|
||||
checkpointJson("with-dep/substituter", substituter);
|
||||
|
||||
// Snapshot the destination store before (should be empty)
|
||||
checkpointJson("../dummy-store/empty", dummyStore);
|
||||
|
||||
// Neither path should exist in the destination store yet
|
||||
ASSERT_FALSE(dummyStore->isValidPath(dependencyPath));
|
||||
ASSERT_FALSE(dummyStore->isValidPath(mainPath));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituterAsStore = substituter;
|
||||
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||
|
||||
// Create a substitution goal for the main path only
|
||||
// The worker should automatically substitute the dependency as well
|
||||
auto goal = worker.makePathSubstitutionGoal(mainPath);
|
||||
|
||||
// Run the worker
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after (should match the substituter)
|
||||
checkpointJson("with-dep/substituter", dummyStore);
|
||||
|
||||
// Both paths should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(dependencyPath));
|
||||
ASSERT_TRUE(dummyStore->isValidPath(mainPath));
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
}
|
||||
|
||||
TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
|
||||
{
|
||||
// Enable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
|
||||
// Create a CA floating output derivation
|
||||
Derivation drv;
|
||||
drv.name = "test-ca-drv";
|
||||
drv.outputs = {
|
||||
{
|
||||
"out",
|
||||
DerivationOutput{DerivationOutput::CAFloating{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hashAlgo = HashAlgorithm::SHA256,
|
||||
}},
|
||||
},
|
||||
};
|
||||
|
||||
// Write the derivation to the destination store
|
||||
auto drvPath = writeDerivation(*dummyStore, drv);
|
||||
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("ca-drv/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo of the derivation
|
||||
// For CA floating derivations, the kind is Deferred since outputs aren't known until build
|
||||
auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
|
||||
ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto drvHash = hashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object
|
||||
auto outputPath = substituter->addToStore(
|
||||
"test-ca-drv-out",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I am the output of a CA derivation",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Add the realisation (build trace) to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
drvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
UnkeyedRealisation{
|
||||
.outPath = outputPath,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Snapshot the substituter
|
||||
checkpointJson("ca-drv/substituter", substituter);
|
||||
|
||||
// The realisation should not exist in the destination store yet
|
||||
DrvOutput drvOutput{drvHash, "out"};
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituterAsStore = substituter;
|
||||
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||
|
||||
// Create a derivation goal for the CA derivation output
|
||||
// The worker should substitute the output rather than building
|
||||
auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true);
|
||||
|
||||
// Run the worker
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after
|
||||
checkpointJson("ca-drv/store-after", dummyStore);
|
||||
|
||||
// The output path should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(outputPath));
|
||||
|
||||
// The realisation should now exist in the destination store
|
||||
auto realisation = dummyStore->queryRealisation(drvOutput);
|
||||
ASSERT_TRUE(realisation);
|
||||
ASSERT_EQ(realisation->outPath, outputPath);
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
|
||||
// Disable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Test for issue #11928: substituting a CA derivation output should not
|
||||
* require fetching the output of an input derivation when that output
|
||||
* is not referenced.
|
||||
*/
|
||||
TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
|
||||
{
|
||||
// Enable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
|
||||
|
||||
// Create the dependency CA floating derivation
|
||||
Derivation depDrv;
|
||||
depDrv.name = "dep-drv";
|
||||
depDrv.outputs = {
|
||||
{
|
||||
"out",
|
||||
DerivationOutput{DerivationOutput::CAFloating{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hashAlgo = HashAlgorithm::SHA256,
|
||||
}},
|
||||
},
|
||||
};
|
||||
|
||||
// Write the dependency derivation to the destination store
|
||||
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
|
||||
|
||||
// Compute the hash modulo for the dependency derivation
|
||||
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
|
||||
ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto depDrvHash = depHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the dependency in the substituter
|
||||
auto depOutputPath = substituter->addToStore(
|
||||
"dep-drv-out",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents = "I am the dependency output",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// Add the realisation for the dependency to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
depDrvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
UnkeyedRealisation{
|
||||
.outPath = depOutputPath,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Create the root CA floating derivation that depends on depDrv
|
||||
Derivation rootDrv;
|
||||
rootDrv.name = "root-drv";
|
||||
rootDrv.outputs = {
|
||||
{
|
||||
"out",
|
||||
DerivationOutput{DerivationOutput::CAFloating{
|
||||
.method = ContentAddressMethod::Raw::NixArchive,
|
||||
.hashAlgo = HashAlgorithm::SHA256,
|
||||
}},
|
||||
},
|
||||
};
|
||||
// Add the dependency derivation as an input
|
||||
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
|
||||
|
||||
// Write the root derivation to the destination store
|
||||
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
|
||||
|
||||
// Snapshot the destination store before
|
||||
checkpointJson("issue-11928/store-before", dummyStore);
|
||||
|
||||
// Compute the hash modulo for the root derivation
|
||||
auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
|
||||
ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
|
||||
auto rootDrvHash = rootHashModulo.hashes.at("out");
|
||||
|
||||
// Create the output store object for the root derivation
|
||||
// Note: it does NOT reference the dependency's output
|
||||
auto rootOutputPath = substituter->addToStore(
|
||||
"root-drv-out",
|
||||
SourcePath{
|
||||
[] {
|
||||
auto sc = make_ref<MemorySourceAccessor>();
|
||||
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
|
||||
.executable = false,
|
||||
.contents =
|
||||
"I am the root output. "
|
||||
"I don't reference anything because the other derivation's output is just needed at build time.",
|
||||
}};
|
||||
return sc;
|
||||
}(),
|
||||
},
|
||||
ContentAddressMethod::Raw::NixArchive,
|
||||
HashAlgorithm::SHA256);
|
||||
|
||||
// The DrvOutputs for both derivations
|
||||
DrvOutput depDrvOutput{depDrvHash, "out"};
|
||||
DrvOutput rootDrvOutput{rootDrvHash, "out"};
|
||||
|
||||
// Add the realisation for the root derivation to the substituter
|
||||
substituter->buildTrace.insert_or_assign(
|
||||
rootDrvHash,
|
||||
std::map<std::string, UnkeyedRealisation>{
|
||||
{
|
||||
"out",
|
||||
UnkeyedRealisation{
|
||||
.outPath = rootOutputPath,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Snapshot the substituter
|
||||
// Note: it has realisations for both drvs, but only the root's output store object
|
||||
checkpointJson("issue-11928/substituter", substituter);
|
||||
|
||||
// The realisations should not exist in the destination store yet
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput));
|
||||
ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput));
|
||||
|
||||
// Create a worker with our custom substituter
|
||||
Worker worker{*dummyStore, *dummyStore};
|
||||
|
||||
// Override the substituters to use our dummy store substituter
|
||||
ref<Store> substituterAsStore = substituter;
|
||||
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
|
||||
|
||||
// Create a derivation goal for the root derivation output
|
||||
// The worker should substitute the output rather than building
|
||||
auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false);
|
||||
|
||||
// Run the worker
|
||||
Goals goals;
|
||||
goals.insert(upcast_goal(goal));
|
||||
worker.run(goals);
|
||||
|
||||
// Snapshot the destination store after
|
||||
checkpointJson("issue-11928/store-after", dummyStore);
|
||||
|
||||
// The root output path should now exist in the destination store
|
||||
ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath));
|
||||
|
||||
// The root realisation should now exist in the destination store
|
||||
auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput);
|
||||
ASSERT_TRUE(rootRealisation);
|
||||
ASSERT_EQ(rootRealisation->outPath, rootOutputPath);
|
||||
|
||||
// #11928: The dependency's REALISATION should be fetched, because
|
||||
// it is needed to resolve the underlying derivation. Currently the
|
||||
// realisation is not fetched (bug). Once fixed: Change
|
||||
// depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the
|
||||
// ASSERT_EQ
|
||||
auto depRealisation = dummyStore->queryRealisation(depDrvOutput);
|
||||
ASSERT_FALSE(depRealisation);
|
||||
// ASSERT_EQ(depRealisation->outPath, depOutputPath);
|
||||
|
||||
// The dependency's OUTPUT is correctly not fetched (not referenced by root output)
|
||||
ASSERT_FALSE(dummyStore->isValidPath(depOutputPath));
|
||||
|
||||
// Verify the goal succeeded
|
||||
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
|
||||
|
||||
// Disable CA derivations experimental feature
|
||||
experimentalFeatureSettings.set("extra-experimental-features", "");
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -988,7 +988,7 @@ Path DerivationBuildingGoal::openLogFile()
|
||||
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
|
||||
|
||||
if (settings.compressLog)
|
||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
|
||||
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
|
||||
else
|
||||
logSink = logFileSink;
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#include "nix/store/build/derivation-goal.hh"
|
||||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||
#include "nix/store/build/derivation-building-goal.hh"
|
||||
#include "nix/store/build/derivation-resolution-goal.hh"
|
||||
#ifndef _WIN32 // TODO enable build hook on Windows
|
||||
@@ -101,24 +100,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
through substitutes. If that doesn't work, we'll build
|
||||
them. */
|
||||
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
|
||||
if (!checkResult) {
|
||||
DrvOutput id{outputHash, wantedOutput};
|
||||
auto g = worker.makeDrvOutputSubstitutionGoal(id);
|
||||
waitees.insert(g);
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
if (nrFailed == 0) {
|
||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath)));
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed == 0)
|
||||
worker.store.registerDrvOutput({*g->outputInfo, id});
|
||||
else
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
}
|
||||
} else {
|
||||
if (!checkResult)
|
||||
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
|
||||
else {
|
||||
auto * cap = getDerivationCA(*drv);
|
||||
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
|
||||
checkResult->first.outPath,
|
||||
@@ -226,6 +210,11 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
|
||||
.outputName = wantedOutput,
|
||||
}};
|
||||
newRealisation.signatures.clear();
|
||||
if (!drv->type().isFixed()) {
|
||||
auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
|
||||
newRealisation.dependentRealisations =
|
||||
drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
|
||||
}
|
||||
worker.store.signRealisation(newRealisation);
|
||||
worker.store.registerDrvOutput(newRealisation);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/util/callback.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/store/globals.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -19,11 +21,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
trace("init");
|
||||
|
||||
/* If the derivation already exists, we’re done */
|
||||
if ((outputInfo = worker.store.queryRealisation(id))) {
|
||||
if (worker.store.queryRealisation(id)) {
|
||||
co_return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
auto subs = worker.getSubstituters();
|
||||
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
||||
|
||||
bool substituterFailed = false;
|
||||
|
||||
@@ -77,6 +79,12 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
|
||||
worker.childTerminated(this);
|
||||
|
||||
/*
|
||||
* The realisation corresponding to the given output id.
|
||||
* Will be filled once we can get it.
|
||||
*/
|
||||
std::shared_ptr<const UnkeyedRealisation> outputInfo;
|
||||
|
||||
try {
|
||||
outputInfo = promise->get_future().get();
|
||||
} catch (std::exception & e) {
|
||||
@@ -87,6 +95,45 @@ Goal::Co DrvOutputSubstitutionGoal::init()
|
||||
if (!outputInfo)
|
||||
continue;
|
||||
|
||||
bool failed = false;
|
||||
|
||||
Goals waitees;
|
||||
|
||||
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
|
||||
if (depId != id) {
|
||||
if (auto localOutputInfo = worker.store.queryRealisation(depId);
|
||||
localOutputInfo && localOutputInfo->outPath != depPath) {
|
||||
warn(
|
||||
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
|
||||
"Local: %s\n"
|
||||
"Remote: %s",
|
||||
sub->config.getHumanReadableURI(),
|
||||
depId.to_string(),
|
||||
worker.store.printStorePath(localOutputInfo->outPath),
|
||||
worker.store.printStorePath(depPath));
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
|
||||
}
|
||||
}
|
||||
|
||||
if (failed)
|
||||
continue;
|
||||
|
||||
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
|
||||
|
||||
co_await await(std::move(waitees));
|
||||
|
||||
trace("output path substituted");
|
||||
|
||||
if (nrFailed > 0) {
|
||||
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
|
||||
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
|
||||
}
|
||||
|
||||
worker.store.registerDrvOutput({*outputInfo, id});
|
||||
|
||||
trace("finished");
|
||||
co_return amDone(ecSuccess);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/store/nar-info.hh"
|
||||
#include "nix/util/finally.hh"
|
||||
@@ -59,7 +60,7 @@ Goal::Co PathSubstitutionGoal::init()
|
||||
throw Error(
|
||||
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
|
||||
|
||||
auto subs = worker.getSubstituters();
|
||||
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
|
||||
|
||||
bool substituterFailed = false;
|
||||
std::optional<Error> lastStoresException = std::nullopt;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
#include "nix/store/local-store.hh"
|
||||
#include "nix/store/machines.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/store/build/worker.hh"
|
||||
#include "nix/store/build/substitution-goal.hh"
|
||||
#include "nix/store/build/drv-output-substitution-goal.hh"
|
||||
@@ -22,7 +21,6 @@ Worker::Worker(Store & store, Store & evalStore)
|
||||
, actSubstitutions(*logger, actCopyPaths)
|
||||
, store(store)
|
||||
, evalStore(evalStore)
|
||||
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
|
||||
{
|
||||
nrLocalBuilds = 0;
|
||||
nrSubstitutions = 0;
|
||||
|
||||
@@ -341,12 +341,8 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
|
||||
|
||||
if (parsed) {
|
||||
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
|
||||
if (!e)
|
||||
if (!e || !e->is_object())
|
||||
return ret;
|
||||
if (!e->is_object()) {
|
||||
warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring");
|
||||
return ret;
|
||||
}
|
||||
for (auto & [key, storePathsJson] : getObject(*e)) {
|
||||
StringSet ss;
|
||||
flatten(storePathsJson, ss);
|
||||
|
||||
@@ -14,14 +14,11 @@ namespace nix {
|
||||
class Worker;
|
||||
|
||||
/**
|
||||
* Fetch a `Realisation` (drv ⨯ output name -> output path) from a
|
||||
* substituter.
|
||||
*
|
||||
* If the output store object itself should also be substituted, that is
|
||||
* the responsibility of the caller to do so.
|
||||
*
|
||||
* @todo rename this `BuidlTraceEntryGoal`, which will make sense
|
||||
* especially once `Realisation` is renamed to `BuildTraceEntry`.
|
||||
* Substitution of a derivation output.
|
||||
* This is done in three steps:
|
||||
* 1. Fetch the output info from a substituter
|
||||
* 2. Substitute the corresponding output path
|
||||
* 3. Register the output info
|
||||
*/
|
||||
class DrvOutputSubstitutionGoal : public Goal
|
||||
{
|
||||
@@ -34,12 +31,6 @@ class DrvOutputSubstitutionGoal : public Goal
|
||||
public:
|
||||
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
|
||||
|
||||
/**
|
||||
* The realisation corresponding to the given output id.
|
||||
* Will be filled once we can get it.
|
||||
*/
|
||||
std::shared_ptr<const UnkeyedRealisation> outputInfo;
|
||||
|
||||
Co init();
|
||||
|
||||
std::string key() override;
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
#include "nix/store/realisation.hh"
|
||||
#include "nix/util/muxable-pipe.hh"
|
||||
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <thread>
|
||||
|
||||
@@ -172,14 +171,6 @@ public:
|
||||
Store & store;
|
||||
Store & evalStore;
|
||||
|
||||
/**
|
||||
* Function to get the substituters to use for path substitution.
|
||||
*
|
||||
* Defaults to `getDefaultSubstituters`. This allows tests to
|
||||
* inject custom substituters.
|
||||
*/
|
||||
std::function<std::list<ref<Store>>()> getSubstituters;
|
||||
|
||||
#ifndef _WIN32 // TODO Enable building on Windows
|
||||
std::unique_ptr<HookInstance> hook;
|
||||
#endif
|
||||
|
||||
@@ -33,22 +33,6 @@ private:
|
||||
public:
|
||||
PathLocks();
|
||||
PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
|
||||
|
||||
PathLocks(PathLocks && other) noexcept
|
||||
: fds(std::exchange(other.fds, {}))
|
||||
, deletePaths(other.deletePaths)
|
||||
{
|
||||
}
|
||||
|
||||
PathLocks & operator=(PathLocks && other) noexcept
|
||||
{
|
||||
fds = std::exchange(other.fds, {});
|
||||
deletePaths = other.deletePaths;
|
||||
return *this;
|
||||
}
|
||||
|
||||
PathLocks(const PathLocks &) = delete;
|
||||
PathLocks & operator=(const PathLocks &) = delete;
|
||||
bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
|
||||
~PathLocks();
|
||||
void unlock();
|
||||
@@ -61,10 +45,6 @@ struct FdLock
|
||||
bool acquired = false;
|
||||
|
||||
FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
|
||||
FdLock(const FdLock &) = delete;
|
||||
FdLock & operator=(const FdLock &) = delete;
|
||||
FdLock(FdLock &&) = delete;
|
||||
FdLock & operator=(FdLock &&) = delete;
|
||||
|
||||
~FdLock()
|
||||
{
|
||||
|
||||
@@ -56,6 +56,14 @@ struct UnkeyedRealisation
|
||||
|
||||
StringSet signatures;
|
||||
|
||||
/**
|
||||
* The realisations that are required for the current one to be valid.
|
||||
*
|
||||
* When importing this realisation, the store will first check that all its
|
||||
* dependencies exist, and map to the correct output path
|
||||
*/
|
||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
||||
|
||||
std::string fingerprint(const DrvOutput & key) const;
|
||||
|
||||
void sign(const DrvOutput & key, const Signer &);
|
||||
@@ -79,6 +87,10 @@ struct Realisation : UnkeyedRealisation
|
||||
|
||||
bool isCompatibleWith(const UnkeyedRealisation & other) const;
|
||||
|
||||
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
|
||||
|
||||
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
|
||||
|
||||
bool operator==(const Realisation &) const = default;
|
||||
auto operator<=>(const Realisation &) const = default;
|
||||
};
|
||||
@@ -142,6 +154,10 @@ struct RealisedPath
|
||||
*/
|
||||
const StorePath & path() const &;
|
||||
|
||||
void closure(Store & store, Set & ret) const;
|
||||
static void closure(Store & store, const Set & startPaths, Set & ret);
|
||||
Set closure(Store & store) const;
|
||||
|
||||
bool operator==(const RealisedPath &) const = default;
|
||||
auto operator<=>(const RealisedPath &) const = default;
|
||||
};
|
||||
|
||||
@@ -1007,6 +1007,9 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
|
||||
|
||||
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
|
||||
|
||||
std::map<DrvOutput, StorePath>
|
||||
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
|
||||
|
||||
template<>
|
||||
struct json_avoids_null<TrustedFlag> : std::true_type
|
||||
{};
|
||||
|
||||
@@ -110,6 +110,8 @@ struct LocalStore::State::Stmts
|
||||
SQLiteStmt QueryAllRealisedOutputs;
|
||||
SQLiteStmt QueryPathFromHashPart;
|
||||
SQLiteStmt QueryValidPaths;
|
||||
SQLiteStmt QueryRealisationReferences;
|
||||
SQLiteStmt AddRealisationReference;
|
||||
};
|
||||
|
||||
LocalStore::LocalStore(ref<const Config> config)
|
||||
@@ -388,6 +390,21 @@ LocalStore::LocalStore(ref<const Config> config)
|
||||
where drvPath = ?
|
||||
;
|
||||
)");
|
||||
state->stmts->QueryRealisationReferences.create(
|
||||
state->db,
|
||||
R"(
|
||||
select drvPath, outputName from Realisations
|
||||
join RealisationsRefs on realisationReference = Realisations.id
|
||||
where referrer = ?;
|
||||
)");
|
||||
state->stmts->AddRealisationReference.create(
|
||||
state->db,
|
||||
R"(
|
||||
insert or replace into RealisationsRefs (referrer, realisationReference)
|
||||
values (
|
||||
(select id from Realisations where drvPath = ? and outputName = ?),
|
||||
(select id from Realisations where drvPath = ? and outputName = ?));
|
||||
)");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -637,6 +654,25 @@ void LocalStore::registerDrvOutput(const Realisation & info)
|
||||
concatStringsSep(" ", info.signatures))
|
||||
.exec();
|
||||
}
|
||||
for (auto & [outputId, depPath] : info.dependentRealisations) {
|
||||
auto localRealisation = queryRealisationCore_(*state, outputId);
|
||||
if (!localRealisation)
|
||||
throw Error(
|
||||
"unable to register the derivation '%s' as it "
|
||||
"depends on the non existent '%s'",
|
||||
info.id.to_string(),
|
||||
outputId.to_string());
|
||||
if (localRealisation->second.outPath != depPath)
|
||||
throw Error(
|
||||
"unable to register the derivation '%s' as it "
|
||||
"depends on a realisation of '%s' that doesn’t"
|
||||
"match what we have locally",
|
||||
info.id.to_string(),
|
||||
outputId.to_string());
|
||||
state->stmts->AddRealisationReference
|
||||
.use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
|
||||
.exec();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1570,6 +1606,21 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
|
||||
return std::nullopt;
|
||||
auto [realisationDbId, res] = *maybeCore;
|
||||
|
||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
||||
auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
|
||||
while (useRealisationRefs.next()) {
|
||||
auto depId = DrvOutput{
|
||||
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
|
||||
useRealisationRefs.getStr(1),
|
||||
};
|
||||
auto dependentRealisation = queryRealisationCore_(state, depId);
|
||||
assert(dependentRealisation); // Enforced by the db schema
|
||||
auto outputPath = dependentRealisation->second.outPath;
|
||||
dependentRealisations.insert({depId, outputPath});
|
||||
}
|
||||
|
||||
res.dependentRealisations = dependentRealisations;
|
||||
|
||||
return {res};
|
||||
}
|
||||
|
||||
|
||||
@@ -334,6 +334,65 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
|
||||
result);
|
||||
}
|
||||
|
||||
std::map<DrvOutput, StorePath>
|
||||
drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
|
||||
{
|
||||
std::map<DrvOutput, StorePath> res;
|
||||
|
||||
for (const auto & input : inputRealisations) {
|
||||
if (pathReferences.count(input.outPath)) {
|
||||
res.insert({input.id, input.outPath});
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::map<DrvOutput, StorePath>
|
||||
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
|
||||
{
|
||||
auto & evalStore = evalStore_ ? *evalStore_ : store;
|
||||
|
||||
std::set<Realisation> inputRealisations;
|
||||
|
||||
auto accumRealisations = [&](this auto & self,
|
||||
const StorePath & inputDrv,
|
||||
const DerivedPathMap<StringSet>::ChildNode & inputNode) -> void {
|
||||
if (!inputNode.value.empty()) {
|
||||
auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
|
||||
for (const auto & outputName : inputNode.value) {
|
||||
auto outputHash = get(outputHashes, outputName);
|
||||
if (!outputHash)
|
||||
throw Error(
|
||||
"output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
|
||||
DrvOutput key{*outputHash, outputName};
|
||||
auto thisRealisation = store.queryRealisation(key);
|
||||
if (!thisRealisation)
|
||||
throw Error(
|
||||
"output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
|
||||
inputRealisations.insert({*thisRealisation, std::move(key)});
|
||||
}
|
||||
}
|
||||
if (!inputNode.value.empty()) {
|
||||
auto d = makeConstantStorePathRef(inputDrv);
|
||||
for (const auto & [outputName, childNode] : inputNode.childMap) {
|
||||
SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
|
||||
self(
|
||||
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
|
||||
resolveDerivedPath(store, next, evalStore_),
|
||||
childNode);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
|
||||
accumRealisations(inputDrv, inputNode);
|
||||
|
||||
auto info = store.queryPathInfo(outputPath);
|
||||
|
||||
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
|
||||
}
|
||||
|
||||
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
|
||||
{
|
||||
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "nix/store/realisation.hh"
|
||||
#include "nix/store/store-api.hh"
|
||||
#include "nix/util/closure.hh"
|
||||
#include "nix/util/signature/local-keys.hh"
|
||||
#include "nix/util/json-utils.hh"
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -25,6 +26,41 @@ std::string DrvOutput::to_string() const
|
||||
return strHash() + "!" + outputName;
|
||||
}
|
||||
|
||||
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
|
||||
{
|
||||
std::set<Realisation> res;
|
||||
Realisation::closure(store, startOutputs, res);
|
||||
return res;
|
||||
}
|
||||
|
||||
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
|
||||
{
|
||||
auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
|
||||
std::set<Realisation> res;
|
||||
for (auto & [currentDep, _] : current.dependentRealisations) {
|
||||
if (auto currentRealisation = store.queryRealisation(currentDep))
|
||||
res.insert({*currentRealisation, currentDep});
|
||||
else
|
||||
throw Error("Unrealised derivation '%s'", currentDep.to_string());
|
||||
}
|
||||
return res;
|
||||
};
|
||||
|
||||
computeClosure<Realisation>(
|
||||
startOutputs,
|
||||
res,
|
||||
[&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
|
||||
std::promise<std::set<Realisation>> promise;
|
||||
try {
|
||||
auto res = getDeps(current);
|
||||
promise.set_value(res);
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
return processEdges(promise);
|
||||
});
|
||||
}
|
||||
|
||||
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
|
||||
{
|
||||
nlohmann::json serialized = Realisation{*this, key};
|
||||
@@ -63,7 +99,43 @@ const StorePath & RealisedPath::path() const &
|
||||
|
||||
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
|
||||
{
|
||||
return outPath == other.outPath;
|
||||
if (outPath == other.outPath) {
|
||||
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
|
||||
warn(
|
||||
"Encountered a realisation for '%s' with an empty set of "
|
||||
"dependencies. This is likely an artifact from an older Nix. "
|
||||
"I’ll try to fix the realisation if I can",
|
||||
id.to_string());
|
||||
return true;
|
||||
} else if (dependentRealisations == other.dependentRealisations) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret)
|
||||
{
|
||||
// FIXME: This only builds the store-path closure, not the real realisation
|
||||
// closure
|
||||
StorePathSet initialStorePaths, pathsClosure;
|
||||
for (auto & path : startPaths)
|
||||
initialStorePaths.insert(path.path());
|
||||
store.computeFSClosure(initialStorePaths, pathsClosure);
|
||||
ret.insert(startPaths.begin(), startPaths.end());
|
||||
ret.insert(pathsClosure.begin(), pathsClosure.end());
|
||||
}
|
||||
|
||||
void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
|
||||
{
|
||||
RealisedPath::closure(store, {*this}, ret);
|
||||
}
|
||||
|
||||
RealisedPath::Set RealisedPath::closure(Store & store) const
|
||||
{
|
||||
RealisedPath::Set ret;
|
||||
closure(store, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -90,19 +162,27 @@ UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & js
|
||||
if (auto signaturesOpt = optionalValueAt(json, "signatures"))
|
||||
signatures = *signaturesOpt;
|
||||
|
||||
std::map<DrvOutput, StorePath> dependentRealisations;
|
||||
if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations"))
|
||||
for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
|
||||
dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
|
||||
|
||||
return UnkeyedRealisation{
|
||||
.outPath = valueAt(json, "outPath"),
|
||||
.signatures = signatures,
|
||||
.dependentRealisations = dependentRealisations,
|
||||
};
|
||||
}
|
||||
|
||||
void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
|
||||
{
|
||||
auto jsonDependentRealisations = nlohmann::json::object();
|
||||
for (auto & [depId, depOutPath] : r.dependentRealisations)
|
||||
jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
|
||||
json = {
|
||||
{"outPath", r.outPath},
|
||||
{"signatures", r.signatures},
|
||||
// back-compat
|
||||
{"dependentRealisations", json::object()},
|
||||
{"dependentRealisations", jsonDependentRealisations},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -292,7 +292,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
|
||||
next->computeFSClosure(newPaths, closure);
|
||||
for (auto & path : closure)
|
||||
goal.addDependency(path);
|
||||
for (auto & real : newRealisations)
|
||||
for (auto & real : Realisation::closure(*next, newRealisations))
|
||||
goal.addedDrvOutputs.insert(real.id);
|
||||
|
||||
return results;
|
||||
|
||||
+22
-18
@@ -915,21 +915,36 @@ std::map<StorePath, StorePath> copyPaths(
|
||||
SubstituteFlag substitute)
|
||||
{
|
||||
StorePathSet storePaths;
|
||||
std::vector<const Realisation *> realisations;
|
||||
std::set<Realisation> toplevelRealisations;
|
||||
for (auto & path : paths) {
|
||||
storePaths.insert(path.path());
|
||||
if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
|
||||
experimentalFeatureSettings.require(Xp::CaDerivations);
|
||||
realisations.push_back(realisation);
|
||||
toplevelRealisations.insert(*realisation);
|
||||
}
|
||||
}
|
||||
|
||||
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
|
||||
|
||||
try {
|
||||
// Copy the realisations. TODO batch this
|
||||
for (const auto * realisation : realisations)
|
||||
dstStore.registerDrvOutput(*realisation, checkSigs);
|
||||
// Copy the realisation closure
|
||||
processGraph<Realisation>(
|
||||
Realisation::closure(srcStore, toplevelRealisations),
|
||||
[&](const Realisation & current) -> std::set<Realisation> {
|
||||
std::set<Realisation> children;
|
||||
for (const auto & [drvOutput, _] : current.dependentRealisations) {
|
||||
auto currentChild = srcStore.queryRealisation(drvOutput);
|
||||
if (!currentChild)
|
||||
throw Error(
|
||||
"incomplete realisation closure: '%s' is a "
|
||||
"dependency of '%s' but isn't registered",
|
||||
drvOutput.to_string(),
|
||||
current.id.to_string());
|
||||
children.insert({*currentChild, drvOutput});
|
||||
}
|
||||
return children;
|
||||
},
|
||||
[&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
|
||||
} catch (MissingExperimentalFeature & e) {
|
||||
// Don't fail if the remote doesn't support CA derivations is it might
|
||||
// not be within our control to change that, and we might still want
|
||||
@@ -1040,19 +1055,8 @@ void copyClosure(
|
||||
if (&srcStore == &dstStore)
|
||||
return;
|
||||
|
||||
StorePathSet closure0;
|
||||
for (auto & path : paths) {
|
||||
if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
|
||||
closure0.insert(opaquePath->path);
|
||||
}
|
||||
}
|
||||
|
||||
StorePathSet closure1;
|
||||
srcStore.computeFSClosure(closure0, closure1);
|
||||
|
||||
RealisedPath::Set closure = paths;
|
||||
for (auto && path : closure1)
|
||||
closure.insert({std::move(path)});
|
||||
RealisedPath::Set closure;
|
||||
RealisedPath::closure(srcStore, paths, closure);
|
||||
|
||||
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
|
||||
}
|
||||
|
||||
@@ -83,31 +83,6 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Specialization for when we need to do "JSON -> `ref<T>`" in one
|
||||
* direction, but "`const T &` -> JSON" in the other direction.
|
||||
*/
|
||||
template<typename T>
|
||||
void checkpointJson(CharacterizationTest & test, PathView testStem, const ref<T> & got)
|
||||
{
|
||||
using namespace nlohmann;
|
||||
|
||||
auto file = test.goldenMaster(Path{testStem} + ".json");
|
||||
|
||||
json gotJson = static_cast<json>(*got);
|
||||
|
||||
if (testAccept()) {
|
||||
std::filesystem::create_directories(file.parent_path());
|
||||
writeFile(file, gotJson.dump(2) + "\n");
|
||||
ADD_FAILURE() << "Updating golden master " << file;
|
||||
} else {
|
||||
json expectedJson = json::parse(readFile(file));
|
||||
ASSERT_EQ(gotJson, expectedJson);
|
||||
ref<T> expected = adl_serializer<ref<T>>::from_json(expectedJson);
|
||||
ASSERT_EQ(*got, *expected);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mixin class for writing characterization tests for `nlohmann::json`
|
||||
* conversions for a given type.
|
||||
|
||||
@@ -82,7 +82,7 @@ protected:
|
||||
|
||||
void TearDown() override
|
||||
{
|
||||
delTmpDir.reset();
|
||||
delTmpDir.release();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -103,7 +103,8 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
|
||||
|
||||
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
|
||||
{
|
||||
auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true);
|
||||
SourcePath path2 = {
|
||||
makeFSSourceAccessor(std::filesystem::path{}, /*trackLastModified=*/true), CanonPath(absPath(path))};
|
||||
path2.dumpPath(sink, filter);
|
||||
return path2.accessor->getLastModified().value();
|
||||
}
|
||||
|
||||
@@ -54,17 +54,6 @@ std::optional<CanonPath> CanonPath::parent() const
|
||||
return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
|
||||
}
|
||||
|
||||
std::optional<std::pair<CanonPath, std::string_view>> CanonPath::parentAndChild() const
|
||||
{
|
||||
if (isRoot())
|
||||
return std::nullopt;
|
||||
auto slash = path.rfind('/');
|
||||
return {{
|
||||
CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, slash))),
|
||||
std::string_view{path}.substr(slash + 1)
|
||||
}};
|
||||
}
|
||||
|
||||
void CanonPath::pop()
|
||||
{
|
||||
assert(!isRoot());
|
||||
|
||||
+14
-77
@@ -69,54 +69,24 @@ struct ArchiveDecompressionSource : Source
|
||||
}
|
||||
};
|
||||
|
||||
/* These strings are a part of the public API in store parameters and such. Do not change!
|
||||
Happens to match enum names. */
|
||||
#define NIX_FOR_EACH_LA_ALGO(MACRO) \
|
||||
MACRO(bzip2) \
|
||||
MACRO(compress) \
|
||||
MACRO(grzip) \
|
||||
MACRO(gzip) \
|
||||
MACRO(lrzip) \
|
||||
MACRO(lz4) \
|
||||
MACRO(lzip) \
|
||||
MACRO(lzma) \
|
||||
MACRO(lzop) \
|
||||
MACRO(xz) \
|
||||
MACRO(zstd)
|
||||
|
||||
struct ArchiveCompressionSink : CompressionSink
|
||||
{
|
||||
Sink & nextSink;
|
||||
struct archive * archive;
|
||||
|
||||
ArchiveCompressionSink(
|
||||
Sink & nextSink, CompressionAlgo method, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
|
||||
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
|
||||
: nextSink(nextSink)
|
||||
{
|
||||
archive = archive_write_new();
|
||||
if (!archive)
|
||||
throw Error("failed to initialize libarchive");
|
||||
|
||||
auto [addFilter, format] = [method]() -> std::pair<int (*)(struct archive *), const char *> {
|
||||
switch (method) {
|
||||
case CompressionAlgo::none:
|
||||
case CompressionAlgo::brotli:
|
||||
unreachable();
|
||||
#define NIX_DEF_LA_ALGO_CASE(algo) \
|
||||
case CompressionAlgo::algo: \
|
||||
return {archive_write_add_filter_##algo, #algo};
|
||||
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
|
||||
#undef NIX_DEF_LA_ALGO_CASE
|
||||
}
|
||||
unreachable();
|
||||
}();
|
||||
|
||||
check(addFilter(archive), "couldn't initialize compression (%s)");
|
||||
check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
|
||||
check(archive_write_set_format_raw(archive));
|
||||
if (parallel)
|
||||
check(archive_write_set_filter_option(archive, format, "threads", "0"));
|
||||
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
|
||||
if (level != COMPRESSION_LEVEL_DEFAULT)
|
||||
check(archive_write_set_filter_option(archive, format, "compression-level", std::to_string(level).c_str()));
|
||||
check(archive_write_set_filter_option(
|
||||
archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
|
||||
// disable internal buffering
|
||||
check(archive_write_set_bytes_per_block(archive, 0));
|
||||
// disable output padding
|
||||
@@ -319,52 +289,19 @@ struct BrotliCompressionSink : ChunkedCompressionSink
|
||||
}
|
||||
};
|
||||
|
||||
/* Parses a *compression* method into the corresponding enum. This is only used
|
||||
in the *compression* case and user interface. Content-Encoding should not use
|
||||
these. */
|
||||
static CompressionAlgo parseNixCompressionAlgoString(std::string_view method)
|
||||
{
|
||||
static const std::unordered_map<std::string_view, CompressionAlgo> lookupTable = {
|
||||
{"none", CompressionAlgo::none},
|
||||
{"br", CompressionAlgo::brotli},
|
||||
#define NIX_DEF_LA_ALGO_NAME(algo) {#algo, CompressionAlgo::algo},
|
||||
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_NAME)
|
||||
#undef NIX_DEF_LA_ALGO_NAME
|
||||
};
|
||||
|
||||
if (auto it = lookupTable.find(method); it != lookupTable.end())
|
||||
return it->second;
|
||||
|
||||
static const StringSet allNames = [&]() {
|
||||
StringSet res;
|
||||
for (auto & [name, _] : lookupTable)
|
||||
res.emplace(name);
|
||||
return res;
|
||||
}();
|
||||
|
||||
throw UnknownCompressionMethod(
|
||||
Suggestions::bestMatches(allNames, method), "unknown compression method '%s'", method);
|
||||
}
|
||||
|
||||
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
|
||||
{
|
||||
return makeCompressionSink(parseNixCompressionAlgoString(method), nextSink, parallel, level);
|
||||
}
|
||||
|
||||
ref<CompressionSink> makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level)
|
||||
{
|
||||
switch (method) {
|
||||
case CompressionAlgo::none:
|
||||
return make_ref<NoneSink>(nextSink);
|
||||
case CompressionAlgo::brotli:
|
||||
return make_ref<BrotliCompressionSink>(nextSink);
|
||||
/* Everything else is supported via libarchive. */
|
||||
#define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo:
|
||||
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
|
||||
std::vector<std::string> la_supports = {
|
||||
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
|
||||
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
|
||||
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
|
||||
#undef NIX_DEF_LA_ALGO_CASE
|
||||
}
|
||||
unreachable();
|
||||
if (method == "none")
|
||||
return make_ref<NoneSink>(nextSink);
|
||||
else if (method == "br")
|
||||
return make_ref<BrotliCompressionSink>(nextSink);
|
||||
else
|
||||
throw UnknownCompressionMethod("unknown compression method '%s'", method);
|
||||
}
|
||||
|
||||
std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)
|
||||
|
||||
@@ -11,24 +11,6 @@ class AutoRemoveJail
|
||||
bool del;
|
||||
public:
|
||||
AutoRemoveJail(int jid);
|
||||
AutoRemoveJail(const AutoRemoveJail &) = delete;
|
||||
AutoRemoveJail & operator=(const AutoRemoveJail &) = delete;
|
||||
|
||||
AutoRemoveJail(AutoRemoveJail && other) noexcept
|
||||
: jid(other.jid)
|
||||
, del(other.del)
|
||||
{
|
||||
other.cancel();
|
||||
}
|
||||
|
||||
AutoRemoveJail & operator=(AutoRemoveJail && other) noexcept
|
||||
{
|
||||
jid = other.jid;
|
||||
del = other.del;
|
||||
other.cancel();
|
||||
return *this;
|
||||
}
|
||||
|
||||
AutoRemoveJail();
|
||||
~AutoRemoveJail();
|
||||
void cancel();
|
||||
|
||||
+26
-15
@@ -70,7 +70,31 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
|
||||
return dst;
|
||||
}
|
||||
|
||||
void RestoreSink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
|
||||
#ifndef _WIN32
|
||||
void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
|
||||
{
|
||||
if (path.isRoot()) {
|
||||
createDirectory(path);
|
||||
callback(*this, path);
|
||||
return;
|
||||
}
|
||||
|
||||
createDirectory(path);
|
||||
assert(dirFd); // If that's not true the above call must have thrown an exception.
|
||||
|
||||
RestoreSink dirSink{startFsync};
|
||||
dirSink.dstPath = append(dstPath, path);
|
||||
dirSink.dirFd =
|
||||
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
|
||||
if (!dirSink.dirFd)
|
||||
throw SysError("opening directory '%s'", dirSink.dstPath.string());
|
||||
|
||||
callback(dirSink, CanonPath::root);
|
||||
}
|
||||
#endif
|
||||
|
||||
void RestoreSink::createDirectory(const CanonPath & path)
|
||||
{
|
||||
auto p = append(dstPath, path);
|
||||
|
||||
@@ -83,16 +107,6 @@ void RestoreSink::createDirectory(const CanonPath & path, std::optional<Director
|
||||
if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
|
||||
throw SysError("creating directory '%s'", p.string());
|
||||
|
||||
if (callback) {
|
||||
RestoreSink dirSink{startFsync};
|
||||
dirSink.dstPath =
|
||||
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
|
||||
if (!dirSink.dirFd)
|
||||
throw SysError("opening directory '%s'", dirSink.dstPath.string());
|
||||
|
||||
(*callback)(dirSink, CanonPath::root);
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@@ -111,10 +125,7 @@ void RestoreSink::createDirectory(const CanonPath & path, std::optional<Director
|
||||
throw SysError("creating directory '%1%'", p.string());
|
||||
}
|
||||
#endif
|
||||
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
}
|
||||
};
|
||||
|
||||
struct RestoreRegularFile : CreateRegularFileSink
|
||||
{
|
||||
|
||||
@@ -238,13 +238,6 @@ public:
|
||||
return ((std::string_view) path).substr(path.rfind('/') + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Combination of `parent` and `baseName`.
|
||||
*
|
||||
* @note that because of the `std::string_view`, we are borrowing for the base name.
|
||||
*/
|
||||
std::optional<std::pair<CanonPath, std::string_view>> parentAndChild() const;
|
||||
|
||||
bool operator==(const CanonPath & x) const
|
||||
{
|
||||
return path == x.path;
|
||||
|
||||
@@ -16,22 +16,6 @@ struct CompressionSink : BufferedSink, FinishSink
|
||||
using FinishSink::finish;
|
||||
};
|
||||
|
||||
enum class CompressionAlgo {
|
||||
none,
|
||||
brotli,
|
||||
bzip2,
|
||||
compress,
|
||||
grzip,
|
||||
gzip,
|
||||
lrzip,
|
||||
lz4,
|
||||
lzip,
|
||||
lzma,
|
||||
lzop,
|
||||
xz,
|
||||
zstd,
|
||||
};
|
||||
|
||||
std::string decompress(const std::string & method, std::string_view in);
|
||||
|
||||
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
|
||||
@@ -41,9 +25,6 @@ std::string compress(const std::string & method, std::string_view in, const bool
|
||||
ref<CompressionSink>
|
||||
makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
|
||||
|
||||
ref<CompressionSink>
|
||||
makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel = false, int level = -1);
|
||||
|
||||
MakeError(UnknownCompressionMethod, Error);
|
||||
|
||||
MakeError(CompressionError, Error);
|
||||
|
||||
@@ -480,23 +480,8 @@ class AutoUnmount
|
||||
Path path;
|
||||
bool del;
|
||||
public:
|
||||
AutoUnmount();
|
||||
AutoUnmount(Path &);
|
||||
AutoUnmount(const AutoUnmount &) = delete;
|
||||
|
||||
AutoUnmount(AutoUnmount && other) noexcept
|
||||
: path(std::move(other.path))
|
||||
, del(std::exchange(other.del, false))
|
||||
{
|
||||
}
|
||||
|
||||
AutoUnmount & operator=(AutoUnmount && other) noexcept
|
||||
{
|
||||
path = std::move(other.path);
|
||||
del = std::exchange(other.del, false);
|
||||
return *this;
|
||||
}
|
||||
|
||||
AutoUnmount();
|
||||
~AutoUnmount();
|
||||
void cancel();
|
||||
};
|
||||
|
||||
@@ -34,6 +34,8 @@ struct FileSystemObjectSink
|
||||
{
|
||||
virtual ~FileSystemObjectSink() = default;
|
||||
|
||||
virtual void createDirectory(const CanonPath & path) = 0;
|
||||
|
||||
using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
|
||||
|
||||
/**
|
||||
@@ -45,7 +47,11 @@ struct FileSystemObjectSink
|
||||
* freshly created directory. Use this when it's important to disallow any
|
||||
* intermediate path components from being symlinks.
|
||||
*/
|
||||
virtual void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) = 0;
|
||||
virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
|
||||
{
|
||||
createDirectory(path);
|
||||
callback(*this, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* This function in general is no re-entrant. Only one file can be
|
||||
@@ -80,11 +86,7 @@ void copyRecursive(
|
||||
*/
|
||||
struct NullFileSystemObjectSink : FileSystemObjectSink
|
||||
{
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
{
|
||||
if (callback)
|
||||
(*callback)(*this, CanonPath::root);
|
||||
}
|
||||
void createDirectory(const CanonPath & path) override {}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override {}
|
||||
|
||||
@@ -116,7 +118,11 @@ struct RestoreSink : FileSystemObjectSink
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
|
||||
void createDirectory(const CanonPath & path) override;
|
||||
|
||||
#ifndef _WIN32
|
||||
void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
|
||||
#endif
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
|
||||
|
||||
@@ -138,15 +144,9 @@ struct RegularFileSink : FileSystemObjectSink
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
regular = false;
|
||||
if (callback) {
|
||||
NullFileSystemObjectSink s;
|
||||
/* The children of the
|
||||
directory that cannot exist also cannot exist */
|
||||
(*callback)(s, CanonPath::root);
|
||||
}
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
|
||||
@@ -153,7 +153,7 @@ struct MemorySink : FileSystemObjectSink
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
|
||||
void createDirectory(const CanonPath & path) override;
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
|
||||
|
||||
|
||||
@@ -43,36 +43,6 @@ public:
|
||||
|
||||
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
|
||||
|
||||
/**
|
||||
* Create a `PosixSourceAccessor` and `SourcePath` corresponding to
|
||||
* some native path.
|
||||
*
|
||||
* @param Whether the accessor should return a non-null getLastModified.
|
||||
* When true the accessor must be used only by a single thread.
|
||||
*
|
||||
* The `PosixSourceAccessor` is rooted as far up the tree as
|
||||
* possible, (e.g. on Windows it could scoped to a drive like
|
||||
* `C:\`). This allows more `..` parent accessing to work.
|
||||
*
|
||||
* @note When `path` is trusted user input, canonicalize it using
|
||||
* `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc,
|
||||
* as appropriate for the use case. At least weak canonicalization is
|
||||
* required for the `SourcePath` to do anything useful at the location it
|
||||
* points to.
|
||||
*
|
||||
* @note A canonicalizing behavior is not built in `createAtRoot` so that
|
||||
* callers do not accidentally introduce symlink-related security vulnerabilities.
|
||||
* Furthermore, `createAtRoot` does not know whether the file pointed to by
|
||||
* `path` should be resolved if it is itself a symlink. In other words,
|
||||
* `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers.
|
||||
*
|
||||
* See
|
||||
* [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path)
|
||||
* and
|
||||
* [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path).
|
||||
*/
|
||||
static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false);
|
||||
|
||||
std::optional<std::time_t> getLastModified() override
|
||||
{
|
||||
return trackLastModified ? std::optional{mtime} : std::nullopt;
|
||||
|
||||
@@ -159,8 +159,6 @@ struct FdSink : BufferedSink
|
||||
}
|
||||
|
||||
FdSink(FdSink &&) = default;
|
||||
FdSink(const FdSink &) = delete;
|
||||
FdSink & operator=(const FdSink &) = delete;
|
||||
|
||||
FdSink & operator=(FdSink && s)
|
||||
{
|
||||
@@ -202,10 +200,8 @@ struct FdSource : BufferedSource, RestartableSource
|
||||
}
|
||||
|
||||
FdSource(FdSource &&) = default;
|
||||
|
||||
FdSource & operator=(FdSource && s) = default;
|
||||
FdSource(const FdSource &) = delete;
|
||||
FdSource & operator=(const FdSource & s) = delete;
|
||||
~FdSource() = default;
|
||||
|
||||
bool good() override;
|
||||
void restart() override;
|
||||
@@ -456,11 +452,6 @@ struct LambdaSink : Sink
|
||||
{
|
||||
}
|
||||
|
||||
LambdaSink(LambdaSink &&) = delete;
|
||||
LambdaSink(const LambdaSink &) = delete;
|
||||
LambdaSink & operator=(LambdaSink &&) = delete;
|
||||
LambdaSink & operator=(const LambdaSink &) = delete;
|
||||
|
||||
~LambdaSink()
|
||||
{
|
||||
cleanupFun();
|
||||
@@ -637,11 +628,6 @@ struct FramedSource : Source
|
||||
{
|
||||
}
|
||||
|
||||
FramedSource(FramedSource &&) = delete;
|
||||
FramedSource(const FramedSource &) = delete;
|
||||
FramedSource & operator=(FramedSource &&) = delete;
|
||||
FramedSource & operator=(const FramedSource &) = delete;
|
||||
|
||||
~FramedSource()
|
||||
{
|
||||
try {
|
||||
@@ -699,11 +685,6 @@ struct FramedSink : nix::BufferedSink
|
||||
{
|
||||
}
|
||||
|
||||
FramedSink(FramedSink &&) = delete;
|
||||
FramedSink(const FramedSink &) = delete;
|
||||
FramedSink & operator=(FramedSink &&) = delete;
|
||||
FramedSink & operator=(const FramedSink &) = delete;
|
||||
|
||||
~FramedSink()
|
||||
{
|
||||
try {
|
||||
|
||||
@@ -69,10 +69,13 @@ public:
|
||||
{
|
||||
}
|
||||
public:
|
||||
Lock(Lock && l) = delete;
|
||||
Lock(Lock && l)
|
||||
: s(l.s)
|
||||
{
|
||||
unreachable();
|
||||
}
|
||||
|
||||
Lock(const Lock & l) = delete;
|
||||
Lock & operator=(Lock && l) = delete;
|
||||
Lock & operator=(const Lock & l) = delete;
|
||||
|
||||
~Lock() {}
|
||||
|
||||
@@ -107,8 +110,6 @@ public:
|
||||
|
||||
struct WriteLock : Lock<WL>
|
||||
{
|
||||
using Lock<WL>::Lock;
|
||||
|
||||
T * operator->()
|
||||
{
|
||||
return &WriteLock::s->data;
|
||||
@@ -130,8 +131,6 @@ public:
|
||||
|
||||
struct ReadLock : Lock<RL>
|
||||
{
|
||||
using Lock<RL>::Lock;
|
||||
|
||||
const T * operator->()
|
||||
{
|
||||
return &ReadLock::s->data;
|
||||
|
||||
@@ -384,11 +384,6 @@ struct MaintainCount
|
||||
counter += delta;
|
||||
}
|
||||
|
||||
MaintainCount(MaintainCount &&) = delete;
|
||||
MaintainCount(const MaintainCount &) = delete;
|
||||
MaintainCount & operator=(MaintainCount &&) = delete;
|
||||
MaintainCount & operator=(const MaintainCount &) = delete;
|
||||
|
||||
~MaintainCount()
|
||||
{
|
||||
counter -= delta;
|
||||
|
||||
@@ -50,11 +50,6 @@ public:
|
||||
writer.openElement(name, attrs);
|
||||
}
|
||||
|
||||
XMLOpenElement(XMLOpenElement &&) = delete;
|
||||
XMLOpenElement(const XMLOpenElement &) = delete;
|
||||
XMLOpenElement & operator=(XMLOpenElement &&) = delete;
|
||||
XMLOpenElement & operator=(const XMLOpenElement &) = delete;
|
||||
|
||||
~XMLOpenElement()
|
||||
{
|
||||
writer.closeElement();
|
||||
|
||||
@@ -146,7 +146,7 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents
|
||||
|
||||
using File = MemorySourceAccessor::File;
|
||||
|
||||
void MemorySink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
|
||||
void MemorySink::createDirectory(const CanonPath & path)
|
||||
{
|
||||
auto * f = dst.open(path, File{File::Directory{}});
|
||||
if (!f)
|
||||
@@ -154,10 +154,7 @@ void MemorySink::createDirectory(const CanonPath & path, std::optional<Directory
|
||||
|
||||
if (!std::holds_alternative<File::Directory>(f->raw))
|
||||
throw Error("file '%s' is not a directory", path);
|
||||
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
}
|
||||
};
|
||||
|
||||
struct CreateMemoryRegularFile : CreateRegularFileSink
|
||||
{
|
||||
|
||||
@@ -98,13 +98,11 @@ struct NarAccessor : public SourceAccessor
|
||||
}
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
createMember(
|
||||
path,
|
||||
NarMember{.stat = {.type = Type::tDirectory, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
#include "nix/util/posix-source-accessor.hh"
|
||||
#include "nix/util/source-path.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/sync.hh"
|
||||
|
||||
#include <boost/unordered/concurrent_flat_map.hpp>
|
||||
|
||||
@@ -20,15 +18,6 @@ PosixSourceAccessor::PosixSourceAccessor()
|
||||
{
|
||||
}
|
||||
|
||||
SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified)
|
||||
{
|
||||
std::filesystem::path path2 = absPath(path);
|
||||
return {
|
||||
make_ref<PosixSourceAccessor>(path2.root_path(), trackLastModified),
|
||||
CanonPath{path2.relative_path().string()},
|
||||
};
|
||||
}
|
||||
|
||||
std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path)
|
||||
{
|
||||
return root.empty() ? (std::filesystem::path{path.abs()})
|
||||
|
||||
@@ -27,10 +27,6 @@ private:
|
||||
|
||||
public:
|
||||
MonitorFdHup(int fd);
|
||||
MonitorFdHup(MonitorFdHup &&) = delete;
|
||||
MonitorFdHup(const MonitorFdHup &) = delete;
|
||||
MonitorFdHup & operator=(MonitorFdHup &&) = delete;
|
||||
MonitorFdHup & operator=(const MonitorFdHup &) = delete;
|
||||
|
||||
~MonitorFdHup()
|
||||
{
|
||||
|
||||
@@ -141,16 +141,6 @@ struct InterruptCallbackImpl : InterruptCallback
|
||||
{
|
||||
InterruptCallbacks::Token token;
|
||||
|
||||
InterruptCallbackImpl(InterruptCallbacks::Token token)
|
||||
: token(token)
|
||||
{
|
||||
}
|
||||
|
||||
InterruptCallbackImpl(InterruptCallbackImpl &&) = delete;
|
||||
InterruptCallbackImpl(const InterruptCallbackImpl &) = delete;
|
||||
InterruptCallbackImpl & operator=(InterruptCallbackImpl &&) = delete;
|
||||
InterruptCallbackImpl & operator=(const InterruptCallbackImpl &) = delete;
|
||||
|
||||
~InterruptCallbackImpl() override
|
||||
{
|
||||
auto interruptCallbacks(_interruptCallbacks.lock());
|
||||
@@ -163,7 +153,11 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
|
||||
auto interruptCallbacks(_interruptCallbacks.lock());
|
||||
auto token = interruptCallbacks->nextToken++;
|
||||
interruptCallbacks->callbacks.emplace(token, callback);
|
||||
return std::make_unique<InterruptCallbackImpl>(token);
|
||||
|
||||
std::unique_ptr<InterruptCallbackImpl> res{new InterruptCallbackImpl{}};
|
||||
res->token = token;
|
||||
|
||||
return std::unique_ptr<InterruptCallback>(res.release());
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -38,7 +38,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
|
||||
if (!namePart)
|
||||
namePart = baseNameOf(path);
|
||||
|
||||
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
|
||||
SourcePath sourcePath = makeFSSourceAccessor(makeParentCanonical(path));
|
||||
|
||||
auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first
|
||||
: store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path;
|
||||
|
||||
+1
-3
@@ -85,9 +85,7 @@ struct CmdHashBase : Command
|
||||
return std::make_unique<HashSink>(hashAlgo);
|
||||
};
|
||||
|
||||
auto makeSourcePath = [&]() -> SourcePath {
|
||||
return PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
|
||||
};
|
||||
auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(makeParentCanonical(path)); };
|
||||
|
||||
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
|
||||
switch (mode) {
|
||||
|
||||
@@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
|
||||
throw UsageError("unknown flag");
|
||||
|
||||
for (auto & i : opArgs) {
|
||||
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
|
||||
SourcePath sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
|
||||
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath)));
|
||||
}
|
||||
}
|
||||
@@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
|
||||
opArgs.pop_front();
|
||||
|
||||
for (auto & i : opArgs) {
|
||||
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
|
||||
SourcePath sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
|
||||
std::cout << fmt(
|
||||
"%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path));
|
||||
}
|
||||
|
||||
@@ -256,7 +256,7 @@ hashPath(char * algo, int base32, char * path)
|
||||
PPCODE:
|
||||
try {
|
||||
Hash h = hashPath(
|
||||
PosixSourceAccessor::createAtRoot(path),
|
||||
SourcePath{getFSSourceAccessor(), CanonPath(absPath(Path(path)))},
|
||||
FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first;
|
||||
auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
|
||||
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
|
||||
@@ -336,7 +336,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo)
|
||||
auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat;
|
||||
auto path = THIS->store->addToStore(
|
||||
std::string(baseNameOf(srcPath)),
|
||||
PosixSourceAccessor::createAtRoot(srcPath),
|
||||
SourcePath{getFSSourceAccessor(), CanonPath(absPath(Path(srcPath)))},
|
||||
method, parseHashAlgo(algo));
|
||||
XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0)));
|
||||
} catch (Error & e) {
|
||||
|
||||
@@ -25,9 +25,4 @@ nix build -f nondeterministic.nix dep2 --no-link
|
||||
# If everything goes right, we should rebuild dep2 rather than fetch it from
|
||||
# the cache (because that would mean duplicating `current-time` in the closure),
|
||||
# and have `dep1 == dep2`.
|
||||
|
||||
# FIXME: Force the use of small-step resolutions only to fix this in a
|
||||
# better way (#11896, #11928).
|
||||
skipTest "temporarily broken because dependent realisations are removed"
|
||||
|
||||
nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link
|
||||
@@ -22,10 +22,7 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix
|
||||
|
||||
# Restart the build on an empty store, ensuring that we don't build
|
||||
clearStore
|
||||
# FIXME: `dependentCA` should not need to be explicitly mentioned in
|
||||
# this. Force the use of small-step resolutions only to allow not
|
||||
# mentioning it explicitly again. (#11896, #11928).
|
||||
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA
|
||||
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA
|
||||
# Check that the thing we’ve just substituted has its realisation stored
|
||||
nix realisation info --file ./content-addressed.nix transitivelyDependentCA
|
||||
# Check that its dependencies have it too
|
||||
|
||||
@@ -50,16 +50,3 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "
|
||||
# Check it works with the expected structured attrs
|
||||
hacky=$(nix-instantiate --expr "$hackyExpr")
|
||||
nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'
|
||||
|
||||
# Test warning for non-object exportReferencesGraph in structured attrs
|
||||
# shellcheck disable=SC2016
|
||||
expectStderr 0 nix-build --expr '
|
||||
with import ./config.nix;
|
||||
mkDerivation {
|
||||
name = "export-graph-non-object";
|
||||
__structuredAttrs = true;
|
||||
exportReferencesGraph = [ "foo" "bar" ];
|
||||
builder = "/bin/sh";
|
||||
args = ["-c" "echo foo > ${builtins.placeholder "out"}"];
|
||||
}
|
||||
' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object"
|
||||
Reference in new issue
Block a user