Compare commits

..
Author SHA1 Message Date
Sergei Zimmerman bb5d2e9676 libutil-tests: Add tests for makeFSSourceAccessor
Should be pretty self-explanatory. We didn't really have unit tests
for the filesystem source accessor. Now we do and this will be immensely
useful for implementing a unix-only smarter accessor that doesn't suffer
from TOCTOU on symlinks.
2025-12-16 03:04:12 +03:00
Sergei Zimmerman 8b0026312a treewide: Get rid of PosixSourceAccessor::createAtRoot
We'd like to split out the implementation into Unix/Windows-specific
parts to more easily iterate on improving UNIX accessors to make use
of dirfd-based operations (or even openat2). This should be hidden behind
the appropriate interface and not exposed as a static member function of
the PosixSourceAccessor.
2025-12-16 03:00:24 +03:00
80 changed files with 647 additions and 1749 deletions

No files matched your search

+1 -1
View File
@@ -26,7 +26,7 @@ jobs:
# required to find all branches
fetch-depth: 0
- name: Create backport PRs
uses: korthout/backport-action@c656f5d5851037b2b38fb5db2691a03fa229e3b2 # v4.0.1
uses: korthout/backport-action@d07416681cab29bf2661702f925f020aaa962997 # v3.4.1
id: backport
with:
# Config README: https://github.com/korthout/backport-action#backport-action
+4 -4
View File
@@ -125,13 +125,13 @@ jobs:
cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY
if: ${{ matrix.instrumented }}
- name: Upload coverage reports
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v5
with:
name: coverage-reports
path: coverage-reports/
if: ${{ matrix.instrumented }}
- name: Upload installer tarball
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v5
with:
name: installer-${{matrix.os}}
path: out/*
@@ -164,7 +164,7 @@ jobs:
steps:
- uses: actions/checkout@v6
- name: Download installer tarball
uses: actions/download-artifact@v7
uses: actions/download-artifact@v6
with:
name: installer-${{matrix.os}}
path: out
@@ -174,7 +174,7 @@ jobs:
echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT"
TARBALL_PATH="$(find "$GITHUB_WORKSPACE/out" -name 'nix*.tar.xz' -print | head -n 1)"
echo "tarball-path=file://$TARBALL_PATH" >> "$GITHUB_OUTPUT"
- uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 # v31.9.0
- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
if: ${{ !matrix.experimental-installer }}
with:
install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }}
@@ -3,10 +3,6 @@
This section provides some notes on how to start hacking on Nix.
To get the latest version of Nix from GitHub:
> **Note**
>
> When checking out the repo on Windows, make sure you have the git setting `core.symlinks` enabled, before cloning, as there are symlinks in the repo.
```console
$ git clone https://github.com/NixOS/nix.git
$ cd nix
@@ -1,21 +1,27 @@
{{#include build-trace-entry-v2-fixed.md}}
{{#include build-trace-entry-v1-fixed.md}}
## Examples
### Simple build trace entry
```json
{{#include schema/build-trace-entry-v2/simple.json}}
{{#include schema/build-trace-entry-v1/simple.json}}
```
### Build trace entry with dependencies
```json
{{#include schema/build-trace-entry-v1/with-dependent-realisations.json}}
```
### Build trace entry with signature
```json
{{#include schema/build-trace-entry-v2/with-signature.json}}
{{#include schema/build-trace-entry-v1/with-signature.json}}
```
<!--
## Raw Schema
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v2.json)
-->
[JSON Schema for Build Trace Entry v1](schema/build-trace-entry-v1.json)
-->
+1 -1
View File
@@ -17,7 +17,7 @@ schemas = [
'derivation-v4',
'derivation-options-v1',
'deriving-path-v1',
'build-trace-entry-v2',
'build-trace-entry-v1',
'build-result-v1',
'store-v1',
]
@@ -83,7 +83,7 @@ properties:
description: |
A mapping from output names to their build trace entries.
additionalProperties:
"$ref": "build-trace-entry-v2.yaml"
"$ref": "build-trace-entry-v1.yaml"
failure:
type: object
@@ -1,5 +1,5 @@
"$schema": "http://json-schema.org/draft-04/schema"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v2.json"
"$id": "https://nix.dev/manual/nix/latest/protocols/json/schema/build-trace-entry-v1.json"
title: Build Trace Entry
description: |
A record of a successful build outcome for a specific derivation output.
@@ -11,17 +11,10 @@ description: |
> This JSON format is currently
> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-ca-derivations)
> and subject to change.
Verision history:
- Version 1: Original format
- Version 2: Remove `dependentRealisations`
type: object
required:
- id
- outPath
- dependentRealisations
- signatures
allOf:
- "$ref": "#/$defs/key"
@@ -29,11 +22,9 @@ allOf:
properties:
id: {}
outPath: {}
dependentRealisations: {}
signatures: {}
additionalProperties:
dependentRealisations:
description: deprecated field
type: object
additionalProperties: false
"$defs":
key:
@@ -69,6 +60,7 @@ additionalProperties:
type: object
required:
- outPath
- dependentRealisations
- signatures
properties:
outPath:
@@ -77,6 +69,19 @@ additionalProperties:
description: |
The path to the store object that resulted from building this derivation for the given output name.
dependentRealisations:
type: object
title: Underlying Base Build Trace
description: |
This is for [*derived*](@docroot@/store/build-trace.md#derived) build trace entries to ensure coherence.
Keys are derivation output IDs (same format as the main `id` field).
Values are the store paths that those dependencies resolved to.
As described in the linked section on derived build trace traces, derived build trace entries must be kept in addition and not instead of the underlying base build entries.
This is the set of base build trace entries that this derived build trace is derived from.
(The set is also a map since this miniature base build trace must be coherent, mapping each key to a single value.)
patternProperties:
"^sha256:[0-9a-f]{64}![a-zA-Z_][a-zA-Z0-9_-]*$":
"$ref": "store-path-v1.yaml"
@@ -70,7 +70,7 @@ properties:
"^[A-Za-z0-9+/]{43}=$":
type: object
additionalProperties:
"$ref": "./build-trace-entry-v2.yaml#/$defs/value"
"$ref": "./build-trace-entry-v1.yaml#/$defs/value"
additionalProperties: false
"$defs":
+1 -3
View File
@@ -62,11 +62,9 @@ schemas = [
},
{
'stem' : 'build-trace-entry',
'schema' : schema_dir / 'build-trace-entry-v2.yaml',
'schema' : schema_dir / 'build-trace-entry-v1.yaml',
'files' : [
'simple.json',
# The field is no longer supported, but we want to show that we
# ignore it during parsing.
'with-dependent-realisations.json',
'with-signature.json',
],
-5
View File
@@ -741,11 +741,6 @@ public:
inDebugger = true;
}
DebuggerGuard(DebuggerGuard &&) = delete;
DebuggerGuard(const DebuggerGuard &) = delete;
DebuggerGuard & operator=(DebuggerGuard &&) = delete;
DebuggerGuard & operator=(const DebuggerGuard &) = delete;
~DebuggerGuard()
{
inDebugger = false;
-2
View File
@@ -107,8 +107,6 @@ private:
Bindings & operator=(const Bindings &) = delete;
Bindings & operator=(Bindings &&) = delete;
~Bindings() = default;
friend class BindingsBuilder;
/**
+2 -5
View File
@@ -164,6 +164,8 @@ public:
Value ** elems;
ListBuilder(EvalMemory & mem, size_t size);
// NOTE: Can be noexcept because we are just copying integral values and
// raw pointers.
ListBuilder(ListBuilder && x) noexcept
: size(x.size)
, inlineElems{x.inlineElems[0], x.inlineElems[1]}
@@ -171,11 +173,6 @@ public:
{
}
ListBuilder(const ListBuilder &) = delete;
ListBuilder & operator=(ListBuilder &&) = delete;
ListBuilder & operator=(const ListBuilder &) = delete;
~ListBuilder() = default;
Value *& operator[](size_t n)
{
return elems[n];
+2 -18
View File
@@ -4061,8 +4061,6 @@ static RegisterPrimOp primop_sort({
1. Transitivity
If a is less than b and b is less than c, then it follows that a is less than c.
```nix
comparator a b && comparator b c -> comparator a c
```
@@ -4075,23 +4073,9 @@ static RegisterPrimOp primop_sort({
1. Transitivity of equivalence
First, two values a and b are considered equivalent with respect to the comparator if:
```
!comparator a b && !comparator b a
```
In other words, neither is considered "less than" the other.
Transitivity of equivalence means:
If a is equivalent to b, and b is equivalent to c, then a must also be equivalent to c.
```nix
let
equiv = x: y: (!comparator x y && !comparator y x);
in
equiv a b && equiv b c -> equiv a c
let equiv = a: b: (!comparator a b && !comparator b a); in
equiv a b && equiv b c -> equiv a c
```
If the *comparator* violates any of these properties, then `builtins.sort`
+7 -46
View File
@@ -70,6 +70,12 @@ TEST_F(GitUtilsTest, sink_basic)
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
// TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory
// here
// The sync method does not document this behavior, should probably renamed because it's not very
// general, and I can't imagine that "non-conventional" archives or any other source to be handled by
// this sink.
sink->createDirectory(CanonPath("foo-1.1"));
sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) {
@@ -85,7 +91,7 @@ TEST_F(GitUtilsTest, sink_basic)
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
auto result = repo->dereferenceSingletonDirectory(sink->flush().hash);
auto result = repo->dereferenceSingletonDirectory(sink->flush());
auto accessor = repo->getAccessor(result, {}, getRepoName());
auto entries = accessor->readDirectory(CanonPath::root);
ASSERT_EQ(entries.size(), 5u);
@@ -117,51 +123,6 @@ TEST_F(GitUtilsTest, sink_hardlink)
}
};
TEST_F(GitUtilsTest, sink_root_file)
{
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
sink->createRegularFile(
CanonPath::root, [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
auto result = sink->flush();
ASSERT_EQ(result.mode, nix::git::Mode::Regular);
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
ASSERT_EQ(accessor->readFile(CanonPath::root), "hello world");
};
TEST_F(GitUtilsTest, sink_root_symlink)
{
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
sink->createSymlink(CanonPath::root, "target");
auto result = sink->flush();
ASSERT_EQ(result.mode, nix::git::Mode::Symlink);
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
ASSERT_EQ(accessor->readLink(CanonPath::root), "target");
};
TEST_F(GitUtilsTest, sink_hardlink_to_root)
{
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
sink->createRegularFile(
CanonPath("hello"), [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
sink->createDirectory(CanonPath("subdir"));
sink->createHardlink(CanonPath("subdir/link"), CanonPath("hello"));
auto result = sink->flush();
ASSERT_EQ(result.mode, nix::git::Mode::Directory);
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
ASSERT_EQ(accessor->readFile(CanonPath("subdir/link")), "hello world");
};
TEST_F(GitUtilsTest, peel_reference)
{
// Create a commit in the repo
+99 -192
View File
@@ -962,15 +962,15 @@ struct GitSourceAccessor : SourceAccessor
Blob getBlob(State & state, const CanonPath & path, bool expectSymlink)
{
if (!expectSymlink && git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
return dupObject<Blob>((git_blob *) &*state.root);
auto notExpected = [&]() {
throw Error(expectSymlink ? "'%s' is not a symlink" : "'%s' is not a regular file", showPath(path));
};
if (path.isRoot()) {
if (git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
return dupObject<Blob>((git_blob *) &*state.root);
if (path.isRoot())
notExpected();
}
auto entry = need(state, path);
@@ -1054,24 +1054,16 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
}
};
namespace {
struct DirEntry
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
git_oid oid;
git_filemode_t mode;
};
ref<GitRepoImpl> repo;
struct DirectoryState
{
struct PendingDir
{
std::string name;
TreeBuilder builder;
};
GitRepoImpl & repo;
TreeBuilder rootBuilder;
std::vector<PendingDir> pendingDirs;
/**
@@ -1086,9 +1078,30 @@ struct DirectoryState
*/
decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
DirectoryState(GitRepoImpl & repo)
void pushBuilder(std::string name)
{
const git_tree_entry * entry;
Tree prevTree = nullptr;
if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
/* Clone a tree that we've already finished. This happens
if a tarball has directory entries that are not
contiguous. */
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
throw Error("parent of '%s' is not a directory", name);
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
}
git_treebuilder * b;
if (git_treebuilder_new(&b, *repo, prevTree.get()))
throw Error("creating a tree builder: %s", git_error_last()->message);
pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
};
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
: repo(repo)
, rootBuilder(makeBuilder())
{
/* Monkey-patching the pack backend to only read the pack directory
once. Otherwise it will do a readdir for each added oid when it's
@@ -1113,145 +1126,62 @@ struct DirectoryState
*/
packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
}
pushBuilder("");
}
TreeBuilder makeBuilder(git_tree * prevTree = nullptr);
std::vector<std::string> prepareDirs(const CanonPath & path);
void updateBuilders(std::span<const std::string> names);
std::pair<git_oid, std::string> popBuilder();
void pushBuilder(std::string name);
void addToTree(const std::string & name, const DirEntry & entry);
};
TreeBuilder DirectoryState::makeBuilder(git_tree * prevTree)
{
TreeBuilder builder;
if (git_treebuilder_new(Setter(builder), repo, prevTree))
throw Error("creating a tree builder: %s", git_error_last()->message);
return builder;
}
git_oid finishBuilder(TreeBuilder & builder)
{
git_oid oid;
if (git_treebuilder_write(&oid, builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
return oid;
}
void DirectoryState::pushBuilder(std::string name)
{
auto & parentBuilder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
const git_tree_entry * entry;
Tree prevTree = nullptr;
if ((entry = git_treebuilder_get(parentBuilder.get(), name.c_str()))) {
/* Clone a tree that we've already finished. This happens
if a tarball has directory entries that are not
contiguous. */
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
throw Error("parent of '%s' is not a directory", name);
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), repo, entry))
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
}
pendingDirs.push_back({
.name = std::move(name),
.builder = makeBuilder(prevTree.get()),
});
}
std::pair<git_oid, std::string> DirectoryState::popBuilder()
{
assert(!pendingDirs.empty());
auto pending = std::move(pendingDirs.back());
pendingDirs.pop_back();
return {finishBuilder(pending.builder), pending.name};
}
void DirectoryState::addToTree(const std::string & name, const DirEntry & entry)
{
auto & builder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &entry.oid, entry.mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
}
void DirectoryState::updateBuilders(std::span<const std::string> names)
{
// Find the common prefix of pendingDirs and names.
size_t prefixLen = 0;
for (; prefixLen < names.size() && prefixLen < pendingDirs.size(); ++prefixLen)
if (names[prefixLen] != pendingDirs[prefixLen].name)
break;
// Finish the builders that are not part of the common prefix.
for (auto n = pendingDirs.size(); n > prefixLen; --n) {
auto [oid, name] = popBuilder();
addToTree(name, DirEntry{.oid = oid, .mode = GIT_FILEMODE_TREE});
}
// Create builders for the new directories.
for (auto n = prefixLen; n < names.size(); ++n)
pushBuilder(names[n]);
}
std::vector<std::string> DirectoryState::prepareDirs(const CanonPath & path)
{
std::vector<std::string> pathComponents;
for (auto & c : path)
pathComponents.emplace_back(c);
updateBuilders(pathComponents);
return pathComponents;
}
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
ref<GitRepoImpl> repo;
/**
* Root state:
* - nullopt: nothing created yet
* - DirEntry: root is a single non-directory object (file/symlink)
* - DirectoryState: root is a directory with builders
*/
std::optional<std::variant<DirEntry, DirectoryState>> root;
DirectoryState & ensureDirectoryState()
std::pair<git_oid, std::string> popBuilder()
{
if (!root) {
root.emplace<DirectoryState>(*repo);
}
if (auto * dirState = std::get_if<DirectoryState>(&*root)) {
return *dirState;
}
throw Error("cannot create directory entry: root is not a directory");
}
assert(!pendingDirs.empty());
auto pending = std::move(pendingDirs.back());
git_oid oid;
if (git_treebuilder_write(&oid, pending.builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
pendingDirs.pop_back();
return {oid, pending.name};
};
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
: repo(repo)
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
{
}
assert(!pendingDirs.empty());
auto & pending = pendingDirs.back();
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
};
void addEntry(const CanonPath & path, const DirEntry & entry)
void updateBuilders(std::span<const std::string> names)
{
if (auto res = path.parentAndChild()) {
auto & [parent, child] = *res;
auto & dirState = ensureDirectoryState();
dirState.prepareDirs(parent);
dirState.addToTree(std::string(child), entry);
} else {
if (root)
throw Error("root already exists");
root = entry;
// Find the common prefix of pendingDirs and names.
size_t prefixLen = 0;
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
break;
// Finish the builders that are not part of the common prefix.
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
auto [oid, name] = popBuilder();
addToTree(name, oid, GIT_FILEMODE_TREE);
}
// Create builders for the new directories.
for (auto n = prefixLen; n < names.size(); ++n)
pushBuilder(names[n]);
};
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
{
std::span<const std::string> pathComponents2{pathComponents};
updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
return true;
}
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
if (!prepareDirs(pathComponents, false))
return;
using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
if (stream)
stream->free(stream);
@@ -1330,68 +1260,60 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
"creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
}
addEntry(
path,
DirEntry{
.oid = oid,
.mode = crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB,
});
addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
}
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
void createDirectory(const CanonPath & path) override
{
ensureDirectoryState().prepareDirs(path);
if (callback)
(*callback)(*this, path);
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
(void) prepareDirs(pathComponents, true);
}
void createSymlink(const CanonPath & path, const std::string & target) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
if (!prepareDirs(pathComponents, false))
return;
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
addEntry(
path,
DirEntry{
.oid = oid,
.mode = GIT_FILEMODE_LINK,
});
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
}
void createHardlink(const CanonPath & path, const CanonPath & target) override
{
auto & dirState = ensureDirectoryState();
std::vector<std::string> pathComponents;
for (auto & c : path)
pathComponents.emplace_back(c);
auto res = path.parentAndChild();
if (!res)
throw Error("hard link cannot be root file system object");
auto & [parent, name] = *res;
dirState.prepareDirs(parent);
if (!prepareDirs(pathComponents, false))
return;
// We can't just look up the path from the start of the root, since
// some parent directories may not have finished yet, so we compute
// a relative path that helps us find the right git_tree_builder or object.
auto relTarget = parent.makeRelative(target);
auto relTarget = CanonPath(path).parent()->makeRelative(target);
auto dir = dirState.pendingDirs.rbegin();
auto dir = pendingDirs.rbegin();
// For each ../ component at the start, go up one directory.
// CanonPath::makeRelative() always puts all .. elements at the start,
// so they're all handled by this loop:
std::string_view relTargetLeft(relTarget);
while (hasPrefix(relTargetLeft, "../")) {
if (dir == dirState.pendingDirs.rend())
if (dir == pendingDirs.rend())
throw Error("invalid hard link target '%s' for path '%s'", target, path);
++dir;
relTargetLeft = relTargetLeft.substr(3);
}
if (dir == pendingDirs.rend())
throw Error("invalid hard link target '%s' for path '%s'", target, path);
// Look up the remainder of the target, starting at the
// top-most `git_treebuilder`.
std::variant<git_treebuilder *, git_oid> curDir{
dir == dirState.pendingDirs.rend() ? dirState.rootBuilder.get() : dir->builder.get()};
std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
Object tree; // needed to keep `entry` alive
const git_tree_entry * entry = nullptr;
@@ -1411,41 +1333,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
assert(entry);
dirState.addToTree(
std::string(name),
DirEntry{.oid = *git_tree_entry_id(entry), .mode = git_tree_entry_filemode(entry)});
addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
}
git::TreeEntry flush() override
Hash flush() override
{
if (!root)
throw Error("nothing to flush");
updateBuilders({});
auto [oid, mode] = std::visit(
overloaded{
[](DirEntry & entry) -> std::pair<git_oid, git_filemode_t> {
return {entry.oid, entry.mode};
},
[](DirectoryState & dirState) -> std::pair<git_oid, git_filemode_t> {
dirState.updateBuilders({});
return {finishBuilder(dirState.rootBuilder), GIT_FILEMODE_TREE};
},
},
*root);
auto [oid, _name] = popBuilder();
if (auto * backend = repo->packBackend) {
/* We are done writing blobs, can restore refresh functionality. */
backend->refresh = dirState.packfileOdbRefresh;
backend->refresh = packfileOdbRefresh;
}
repo->flush();
return git::TreeEntry{.mode = static_cast<git::Mode>(mode), .hash = toHash(oid)};
return toHash(oid);
}
};
}
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
{
auto self = ref<GitRepoImpl>(shared_from_this());
+2 -2
View File
@@ -321,12 +321,12 @@ struct GitArchiveInputScheme : InputScheme
auto tarballCache = settings.getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto treeEntry = parseSink->flush();
auto tree = parseSink->flush();
act.reset();
TarballInfo tarballInfo{
.treeHash = tarballCache->dereferenceSingletonDirectory(treeEntry.hash), .lastModified = lastModified};
.treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified};
cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});
@@ -2,7 +2,6 @@
#include "nix/fetchers/filtering-source-accessor.hh"
#include "nix/util/fs-sink.hh"
#include "nix/util/git.hh"
namespace nix {
@@ -18,9 +17,9 @@ struct Settings;
struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
{
/**
* Flush builder and return the final Git tree entry (hash and mode).
* Flush builder and return a final Git hash.
*/
virtual git::TreeEntry flush() = 0;
virtual Hash flush() = 0;
};
struct GitAccessorOptions
+2 -2
View File
@@ -182,7 +182,7 @@ static DownloadTarballResult downloadTarball_(
auto tarballCache = settings.getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto treeEntry = parseSink->flush();
auto tree = parseSink->flush();
act.reset();
@@ -196,7 +196,7 @@ static DownloadTarballResult downloadTarball_(
infoAttrs = cached->value;
} else {
infoAttrs.insert_or_assign("etag", res->etag);
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(treeEntry.hash).gitRev());
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(tree).gitRev());
infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
if (res->immutableUrl)
infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);
@@ -88,38 +88,25 @@ public:
}
};
#define VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_read) \
{ \
readProtoTest(STEM, VERSION, VALUE); \
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_read) \
{ \
readProtoTest(STEM, VERSION, VALUE); \
} \
TEST_F(FIXTURE, NAME##_write) \
{ \
writeProtoTest(STEM, VERSION, VALUE); \
}
#define VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_write) \
{ \
writeProtoTest(STEM, VERSION, VALUE); \
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
} \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE)
#define VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
}
#define VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
TEST_F(FIXTURE, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_READ_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
VERSIONED_WRITE_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)
} // namespace nix
+27 -23
View File
@@ -47,30 +47,24 @@ public:
}
};
#define READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME##_read) \
{ \
readProtoTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME##_read) \
{ \
readProtoTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_write) \
{ \
writeProtoTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_json_read) \
{ \
readJsonTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME##_write) \
{ \
writeProtoTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME##_json_write) \
{ \
writeJsonTest(STEM, VALUE); \
}
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
READ_CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
WRITE_CHARACTERIZATION_TEST(NAME, STEM, VALUE)
CHARACTERIZATION_TEST(
string,
"string",
@@ -147,7 +141,7 @@ CHARACTERIZATION_TEST(
},
}))
READ_CHARACTERIZATION_TEST(
CHARACTERIZATION_TEST(
realisation_with_deps,
"realisation-with-deps",
(std::tuple<Realisation>{
@@ -155,6 +149,16 @@ READ_CHARACTERIZATION_TEST(
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
.dependentRealisations =
{
{
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
@@ -1,58 +0,0 @@
{
"buildTrace": {
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
"out": {
"dependentRealisations": {},
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
"signatures": []
}
}
},
"config": {
"store": "/nix/store"
},
"contents": {
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
"contents": {
"contents": "I am the output of a CA derivation",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
"narSize": 152,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "test-ca-drv",
"outputs": {
"out": {
"hashAlgo": "sha256",
"method": "nar"
}
},
"system": "",
"version": 4
}
}
}
@@ -1,27 +0,0 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {
"vvyyj6h5ilinsv4q48q5y5vn7s3hxmhl-test-ca-drv.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "test-ca-drv",
"outputs": {
"out": {
"hashAlgo": "sha256",
"method": "nar"
}
},
"system": "",
"version": 4
}
}
}
@@ -1,39 +0,0 @@
{
"buildTrace": {
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
"out": {
"dependentRealisations": {},
"outPath": "hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out",
"signatures": []
}
}
},
"config": {
"store": "/nix/store"
},
"contents": {
"hrva7l0gsk67wffmks761mv4ks4vzsx7-test-ca-drv-out": {
"contents": {
"contents": "I am the output of a CA derivation",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-l0+gmYB0AK65UWuoSh7AbVRI4rAc5/VGqzBGTHgMsiU=",
"narSize": 152,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}
@@ -1,83 +0,0 @@
{
"buildTrace": {
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
"out": {
"dependentRealisations": {},
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
"signatures": []
}
}
},
"config": {
"store": "/nix/store"
},
"contents": {
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
"contents": {
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
"narSize": 232,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
"dynamicOutputs": {},
"outputs": [
"out"
]
}
},
"srcs": []
},
"name": "root-drv",
"outputs": {
"out": {
"hashAlgo": "sha256",
"method": "nar"
}
},
"system": "",
"version": 4
},
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "dep-drv",
"outputs": {
"out": {
"hashAlgo": "sha256",
"method": "nar"
}
},
"system": "",
"version": 4
}
}
}
@@ -1,52 +0,0 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {},
"derivations": {
"11yvkl84ashq63ilwc2mi4va41z2disw-root-drv.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
"dynamicOutputs": {},
"outputs": [
"out"
]
}
},
"srcs": []
},
"name": "root-drv",
"outputs": {
"out": {
"hashAlgo": "sha256",
"method": "nar"
}
},
"system": "",
"version": 4
},
"vy7j6m6p5y0327fhk3zxn12hbpzkh6lp-dep-drv.drv": {
"args": [],
"builder": "",
"env": {},
"inputs": {
"drvs": {},
"srcs": []
},
"name": "dep-drv",
"outputs": {
"out": {
"hashAlgo": "sha256",
"method": "nar"
}
},
"system": "",
"version": 4
}
}
}
@@ -1,68 +0,0 @@
{
"buildTrace": {
"8vEkprm3vQ3BE6JLB8XKfU+AdAwEFOMI/skzyj3pr5I=": {
"out": {
"dependentRealisations": {},
"outPath": "px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out",
"signatures": []
}
},
"gnRuK+wfbXqRPzgO5MyiBebXrV10Kzv+tkZCEuPm7pY=": {
"out": {
"dependentRealisations": {},
"outPath": "w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out",
"signatures": []
}
}
},
"config": {
"store": "/nix/store"
},
"contents": {
"px7apdw6ydm9ynjy5g0bpdcylw3xz2kj-root-drv-out": {
"contents": {
"contents": "I am the root output. I don't reference anything because the other derivation's output is just needed at build time.",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-0mlhg9y1FGb7YsHAsNOmtuW44b8TfoPaNPK6SjVYe5s=",
"narSize": 232,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
},
"w0yjpwh59kpbyc7hz9jgmi44r9br908i-dep-drv-out": {
"contents": {
"contents": "I am the dependency output",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-HK2LBzSTtwuRjc44PH3Ac1JHHPKmfnAgNxz6I5mVgL8=",
"narSize": 144,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}
@@ -1,31 +0,0 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {
"axqic2q30v0sqvcpiqxs139q8w6zd4n8-hello": {
"contents": {
"contents": "Hello, world!",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-KShIJtIwWG1gMSpvPMt5drppc1h5WMwHWzVpNJiVqGI=",
"narSize": 128,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}
@@ -1,55 +0,0 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
"contents": {
"contents": "I am a dependency",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
"narSize": 136,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
},
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
"contents": {
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
"narSize": 184,
"references": [
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}
@@ -1,55 +0,0 @@
{
"buildTrace": {},
"config": {
"store": "/nix/store"
},
"contents": {
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency": {
"contents": {
"contents": "I am a dependency",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-miJnClL0Ai/HAmX1G/pz7P2TIaeFjP5D/VN1rhYf354=",
"narSize": 136,
"references": [],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
},
"k09ldq9fvxb6vfwq0cmv6j1jgqx08y1n-main": {
"contents": {
"contents": "I depend on /nix/store/4k79i02avcckr96r97lqnswn75fi1gv7-dependency",
"executable": false,
"type": "regular"
},
"info": {
"ca": {
"hash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
"method": "nar"
},
"deriver": null,
"narHash": "sha256-CBfMK3HkqiXjpI8HNL1spWD/US4RnQHwI67Ojl50XoQ=",
"narSize": 184,
"references": [
"4k79i02avcckr96r97lqnswn75fi1gv7-dependency"
],
"registrationTime": null,
"signatures": [],
"storeDir": "/nix/store",
"ultimate": false,
"version": 2
}
}
},
"derivations": {}
}
-1
View File
@@ -85,7 +85,6 @@ sources = files(
'store-reference.cc',
'uds-remote-store.cc',
'worker-protocol.cc',
'worker-substitution.cc',
'write-derivation.cc',
)
+45 -36
View File
@@ -44,45 +44,54 @@ TEST_P(RealisationJsonTest, to_json)
writeJsonTest(name, value);
}
Realisation simple{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
},
{
.drvHash = Hash::parseExplicitFormatUnprefixed(
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
HashAlgorithm::SHA256,
HashFormat::Base16),
.outputName = "foo",
},
};
INSTANTIATE_TEST_SUITE_P(
RealisationJSON,
RealisationJsonTest,
::testing::Values(
std::pair{
"simple",
simple,
},
std::pair{
"with-signature",
[&] {
auto r = simple;
// FIXME actually sign properly
r.signatures = {"asdfasdfasdf"};
return r;
}(),
}));
([] {
Realisation simple{
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
},
{
.drvHash = Hash::parseExplicitFormatUnprefixed(
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
HashAlgorithm::SHA256,
HashFormat::Base16),
.outputName = "foo",
},
};
return ::testing::Values(
std::pair{
"simple",
simple,
},
std::pair{
"with-signature",
[&] {
auto r = simple;
// FIXME actually sign properly
r.signatures = {"asdfasdfasdf"};
return r;
}()},
std::pair{
"with-dependent-realisations",
[&] {
auto r = simple;
r.dependentRealisations = {{
{
.drvHash = Hash::parseExplicitFormatUnprefixed(
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
HashAlgorithm::SHA256,
HashFormat::Base16),
.outputName = "foo",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"},
}};
return r;
}(),
});
}
/**
* We no longer have a notion of "dependent realisations", but we still
* want to parse old realisation files. So make this just be a read test
* (no write direction), accordingly.
*/
TEST_F(RealisationTest, dependent_realisations_from_json)
{
readJsonTest("with-dependent-realisations", simple);
}
()));
} // namespace nix
+11 -1
View File
@@ -118,7 +118,7 @@ VERSIONED_CHARACTERIZATION_TEST(
},
}))
VERSIONED_READ_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
realisation_with_deps,
"realisation-with-deps",
@@ -128,6 +128,16 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
.dependentRealisations =
{
{
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
+11 -1
View File
@@ -171,7 +171,7 @@ VERSIONED_CHARACTERIZATION_TEST(
},
}))
VERSIONED_READ_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
realisation_with_deps,
"realisation-with-deps",
@@ -181,6 +181,16 @@ VERSIONED_READ_CHARACTERIZATION_TEST(
{
.outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
.signatures = {"asdf", "qwer"},
.dependentRealisations =
{
{
DrvOutput{
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"},
},
},
},
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
-450
View File
@@ -1,450 +0,0 @@
#include <gtest/gtest.h>
#include <nlohmann/json.hpp>
#include "nix/store/build/worker.hh"
#include "nix/store/derivations.hh"
#include "nix/store/dummy-store-impl.hh"
#include "nix/store/globals.hh"
#include "nix/util/memory-source-accessor.hh"
#include "nix/store/tests/libstore.hh"
#include "nix/util/tests/json-characterization.hh"
namespace nix {
class WorkerSubstitutionTest : public LibStoreTest, public JsonCharacterizationTest<ref<DummyStore>>
{
std::filesystem::path unitTestData = getUnitTestData() / "worker-substitution";
protected:
ref<DummyStore> dummyStore;
ref<DummyStore> substituter;
WorkerSubstitutionTest()
: LibStoreTest([] {
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
config->readOnly = false;
return config->openDummyStore();
}())
, dummyStore(store.dynamic_pointer_cast<DummyStore>())
, substituter([] {
auto config = make_ref<DummyStoreConfig>(DummyStoreConfig::Params{});
config->readOnly = false;
config->isTrusted = true;
return config->openDummyStore();
}())
{
}
public:
std::filesystem::path goldenMaster(std::string_view testStem) const override
{
return unitTestData / testStem;
}
static void SetUpTestSuite()
{
initLibStore(false);
}
};
TEST_F(WorkerSubstitutionTest, singleStoreObject)
{
// Add a store path to the substituter
auto pathInSubstituter = substituter->addToStore(
"hello",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "Hello, world!",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
// Snapshot the substituter (has one store object)
checkpointJson("single/substituter", substituter);
// Snapshot the destination store before (should be empty)
checkpointJson("../dummy-store/empty", dummyStore);
// The path should not exist in the destination store yet
ASSERT_FALSE(dummyStore->isValidPath(pathInSubstituter));
// Create a worker with our custom substituter
Worker worker{*dummyStore, *dummyStore};
// Override the substituters to use our dummy store substituter
ref<Store> substituerAsStore = substituter;
worker.getSubstituters = [substituerAsStore]() -> std::list<ref<Store>> { return {substituerAsStore}; };
// Create a substitution goal for the path
auto goal = worker.makePathSubstitutionGoal(pathInSubstituter);
// Run the worker with -j0 semantics (no local builds, only substitution)
// The worker.run() takes a set of goals
Goals goals;
goals.insert(upcast_goal(goal));
worker.run(goals);
// Snapshot the destination store after (should match the substituter)
checkpointJson("single/substituter", dummyStore);
// The path should now exist in the destination store
ASSERT_TRUE(dummyStore->isValidPath(pathInSubstituter));
// Verify the goal succeeded
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
}
TEST_F(WorkerSubstitutionTest, singleRootStoreObjectWithSingleDepStoreObject)
{
// First, add a dependency store path to the substituter
auto dependencyPath = substituter->addToStore(
"dependency",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "I am a dependency",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
// Now add a store path that references the dependency
auto mainPath = substituter->addToStore(
"main",
SourcePath{
[&] {
auto sc = make_ref<MemorySourceAccessor>();
// Include a reference to the dependency path in the contents
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "I depend on " + substituter->printStorePath(dependencyPath),
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256,
StorePathSet{dependencyPath});
// Snapshot the substituter (has two store objects)
checkpointJson("with-dep/substituter", substituter);
// Snapshot the destination store before (should be empty)
checkpointJson("../dummy-store/empty", dummyStore);
// Neither path should exist in the destination store yet
ASSERT_FALSE(dummyStore->isValidPath(dependencyPath));
ASSERT_FALSE(dummyStore->isValidPath(mainPath));
// Create a worker with our custom substituter
Worker worker{*dummyStore, *dummyStore};
// Override the substituters to use our dummy store substituter
ref<Store> substituterAsStore = substituter;
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
// Create a substitution goal for the main path only
// The worker should automatically substitute the dependency as well
auto goal = worker.makePathSubstitutionGoal(mainPath);
// Run the worker
Goals goals;
goals.insert(upcast_goal(goal));
worker.run(goals);
// Snapshot the destination store after (should match the substituter)
checkpointJson("with-dep/substituter", dummyStore);
// Both paths should now exist in the destination store
ASSERT_TRUE(dummyStore->isValidPath(dependencyPath));
ASSERT_TRUE(dummyStore->isValidPath(mainPath));
// Verify the goal succeeded
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
}
TEST_F(WorkerSubstitutionTest, floatingDerivationOutput)
{
// Enable CA derivations experimental feature
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
// Create a CA floating output derivation
Derivation drv;
drv.name = "test-ca-drv";
drv.outputs = {
{
"out",
DerivationOutput{DerivationOutput::CAFloating{
.method = ContentAddressMethod::Raw::NixArchive,
.hashAlgo = HashAlgorithm::SHA256,
}},
},
};
// Write the derivation to the destination store
auto drvPath = writeDerivation(*dummyStore, drv);
// Snapshot the destination store before
checkpointJson("ca-drv/store-before", dummyStore);
// Compute the hash modulo of the derivation
// For CA floating derivations, the kind is Deferred since outputs aren't known until build
auto hashModulo = hashDerivationModulo(*dummyStore, drv, true);
ASSERT_EQ(hashModulo.kind, DrvHash::Kind::Deferred);
auto drvHash = hashModulo.hashes.at("out");
// Create the output store object
auto outputPath = substituter->addToStore(
"test-ca-drv-out",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "I am the output of a CA derivation",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
// Add the realisation (build trace) to the substituter
substituter->buildTrace.insert_or_assign(
drvHash,
std::map<std::string, UnkeyedRealisation>{
{
"out",
UnkeyedRealisation{
.outPath = outputPath,
},
},
});
// Snapshot the substituter
checkpointJson("ca-drv/substituter", substituter);
// The realisation should not exist in the destination store yet
DrvOutput drvOutput{drvHash, "out"};
ASSERT_FALSE(dummyStore->queryRealisation(drvOutput));
// Create a worker with our custom substituter
Worker worker{*dummyStore, *dummyStore};
// Override the substituters to use our dummy store substituter
ref<Store> substituterAsStore = substituter;
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
// Create a derivation goal for the CA derivation output
// The worker should substitute the output rather than building
auto goal = worker.makeDerivationGoal(drvPath, drv, "out", bmNormal, true);
// Run the worker
Goals goals;
goals.insert(upcast_goal(goal));
worker.run(goals);
// Snapshot the destination store after
checkpointJson("ca-drv/store-after", dummyStore);
// The output path should now exist in the destination store
ASSERT_TRUE(dummyStore->isValidPath(outputPath));
// The realisation should now exist in the destination store
auto realisation = dummyStore->queryRealisation(drvOutput);
ASSERT_TRUE(realisation);
ASSERT_EQ(realisation->outPath, outputPath);
// Verify the goal succeeded
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
// Disable CA derivations experimental feature
experimentalFeatureSettings.set("extra-experimental-features", "");
}
/**
* Test for issue #11928: substituting a CA derivation output should not
* require fetching the output of an input derivation when that output
* is not referenced.
*/
TEST_F(WorkerSubstitutionTest, floatingDerivationOutputWithDepDrv)
{
// Enable CA derivations experimental feature
experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations");
// Create the dependency CA floating derivation
Derivation depDrv;
depDrv.name = "dep-drv";
depDrv.outputs = {
{
"out",
DerivationOutput{DerivationOutput::CAFloating{
.method = ContentAddressMethod::Raw::NixArchive,
.hashAlgo = HashAlgorithm::SHA256,
}},
},
};
// Write the dependency derivation to the destination store
auto depDrvPath = writeDerivation(*dummyStore, depDrv);
// Compute the hash modulo for the dependency derivation
auto depHashModulo = hashDerivationModulo(*dummyStore, depDrv, true);
ASSERT_EQ(depHashModulo.kind, DrvHash::Kind::Deferred);
auto depDrvHash = depHashModulo.hashes.at("out");
// Create the output store object for the dependency in the substituter
auto depOutputPath = substituter->addToStore(
"dep-drv-out",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents = "I am the dependency output",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
// Add the realisation for the dependency to the substituter
substituter->buildTrace.insert_or_assign(
depDrvHash,
std::map<std::string, UnkeyedRealisation>{
{
"out",
UnkeyedRealisation{
.outPath = depOutputPath,
},
},
});
// Create the root CA floating derivation that depends on depDrv
Derivation rootDrv;
rootDrv.name = "root-drv";
rootDrv.outputs = {
{
"out",
DerivationOutput{DerivationOutput::CAFloating{
.method = ContentAddressMethod::Raw::NixArchive,
.hashAlgo = HashAlgorithm::SHA256,
}},
},
};
// Add the dependency derivation as an input
rootDrv.inputDrvs = {.map = {{depDrvPath, {.value = {"out"}}}}};
// Write the root derivation to the destination store
auto rootDrvPath = writeDerivation(*dummyStore, rootDrv);
// Snapshot the destination store before
checkpointJson("issue-11928/store-before", dummyStore);
// Compute the hash modulo for the root derivation
auto rootHashModulo = hashDerivationModulo(*dummyStore, rootDrv, true);
ASSERT_EQ(rootHashModulo.kind, DrvHash::Kind::Deferred);
auto rootDrvHash = rootHashModulo.hashes.at("out");
// Create the output store object for the root derivation
// Note: it does NOT reference the dependency's output
auto rootOutputPath = substituter->addToStore(
"root-drv-out",
SourcePath{
[] {
auto sc = make_ref<MemorySourceAccessor>();
sc->root = MemorySourceAccessor::File{MemorySourceAccessor::File::Regular{
.executable = false,
.contents =
"I am the root output. "
"I don't reference anything because the other derivation's output is just needed at build time.",
}};
return sc;
}(),
},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256);
// The DrvOutputs for both derivations
DrvOutput depDrvOutput{depDrvHash, "out"};
DrvOutput rootDrvOutput{rootDrvHash, "out"};
// Add the realisation for the root derivation to the substituter
substituter->buildTrace.insert_or_assign(
rootDrvHash,
std::map<std::string, UnkeyedRealisation>{
{
"out",
UnkeyedRealisation{
.outPath = rootOutputPath,
},
},
});
// Snapshot the substituter
// Note: it has realisations for both drvs, but only the root's output store object
checkpointJson("issue-11928/substituter", substituter);
// The realisations should not exist in the destination store yet
ASSERT_FALSE(dummyStore->queryRealisation(depDrvOutput));
ASSERT_FALSE(dummyStore->queryRealisation(rootDrvOutput));
// Create a worker with our custom substituter
Worker worker{*dummyStore, *dummyStore};
// Override the substituters to use our dummy store substituter
ref<Store> substituterAsStore = substituter;
worker.getSubstituters = [substituterAsStore]() -> std::list<ref<Store>> { return {substituterAsStore}; };
// Create a derivation goal for the root derivation output
// The worker should substitute the output rather than building
auto goal = worker.makeDerivationGoal(rootDrvPath, rootDrv, "out", bmNormal, false);
// Run the worker
Goals goals;
goals.insert(upcast_goal(goal));
worker.run(goals);
// Snapshot the destination store after
checkpointJson("issue-11928/store-after", dummyStore);
// The root output path should now exist in the destination store
ASSERT_TRUE(dummyStore->isValidPath(rootOutputPath));
// The root realisation should now exist in the destination store
auto rootRealisation = dummyStore->queryRealisation(rootDrvOutput);
ASSERT_TRUE(rootRealisation);
ASSERT_EQ(rootRealisation->outPath, rootOutputPath);
// #11928: The dependency's REALISATION should be fetched, because
// it is needed to resolve the underlying derivation. Currently the
// realisation is not fetched (bug). Once fixed: Change
// depRealisation ASSERT_FALSE to ASSERT_TRUE and uncomment the
// ASSERT_EQ
auto depRealisation = dummyStore->queryRealisation(depDrvOutput);
ASSERT_FALSE(depRealisation);
// ASSERT_EQ(depRealisation->outPath, depOutputPath);
// The dependency's OUTPUT is correctly not fetched (not referenced by root output)
ASSERT_FALSE(dummyStore->isValidPath(depOutputPath));
// Verify the goal succeeded
ASSERT_EQ(upcast_goal(goal)->exitCode, Goal::ecSuccess);
// Disable CA derivations experimental feature
experimentalFeatureSettings.set("extra-experimental-features", "");
}
} // namespace nix
@@ -988,7 +988,7 @@ Path DerivationBuildingGoal::openLogFile()
logFileSink = std::make_shared<FdSink>(fdLogFile.get());
if (settings.compressLog)
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink(CompressionAlgo::bzip2, *logFileSink));
logSink = std::shared_ptr<CompressionSink>(makeCompressionSink("bzip2", *logFileSink));
else
logSink = logFileSink;
+8 -19
View File
@@ -1,5 +1,4 @@
#include "nix/store/build/derivation-goal.hh"
#include "nix/store/build/drv-output-substitution-goal.hh"
#include "nix/store/build/derivation-building-goal.hh"
#include "nix/store/build/derivation-resolution-goal.hh"
#ifndef _WIN32 // TODO enable build hook on Windows
@@ -101,24 +100,9 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && drvOptions.substitutesAllowed()) {
if (!checkResult) {
DrvOutput id{outputHash, wantedOutput};
auto g = worker.makeDrvOutputSubstitutionGoal(id);
waitees.insert(g);
co_await await(std::move(waitees));
if (nrFailed == 0) {
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(g->outputInfo->outPath)));
co_await await(std::move(waitees));
trace("output path substituted");
if (nrFailed == 0)
worker.store.registerDrvOutput({*g->outputInfo, id});
else
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
}
} else {
if (!checkResult)
waitees.insert(upcast_goal(worker.makeDrvOutputSubstitutionGoal(DrvOutput{outputHash, wantedOutput})));
else {
auto * cap = getDerivationCA(*drv);
waitees.insert(upcast_goal(worker.makePathSubstitutionGoal(
checkResult->first.outPath,
@@ -226,6 +210,11 @@ Goal::Co DerivationGoal::haveDerivation(bool storeDerivation)
.outputName = wantedOutput,
}};
newRealisation.signatures.clear();
if (!drv->type().isFixed()) {
auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store;
newRealisation.dependentRealisations =
drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore);
}
worker.store.signRealisation(newRealisation);
worker.store.registerDrvOutput(newRealisation);
}
@@ -3,6 +3,8 @@
#include "nix/store/build/worker.hh"
#include "nix/store/build/substitution-goal.hh"
#include "nix/util/callback.hh"
#include "nix/store/store-open.hh"
#include "nix/store/globals.hh"
namespace nix {
@@ -19,11 +21,11 @@ Goal::Co DrvOutputSubstitutionGoal::init()
trace("init");
/* If the derivation already exists, we’re done */
if ((outputInfo = worker.store.queryRealisation(id))) {
if (worker.store.queryRealisation(id)) {
co_return amDone(ecSuccess);
}
auto subs = worker.getSubstituters();
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
bool substituterFailed = false;
@@ -77,6 +79,12 @@ Goal::Co DrvOutputSubstitutionGoal::init()
worker.childTerminated(this);
/*
* The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const UnkeyedRealisation> outputInfo;
try {
outputInfo = promise->get_future().get();
} catch (std::exception & e) {
@@ -87,6 +95,45 @@ Goal::Co DrvOutputSubstitutionGoal::init()
if (!outputInfo)
continue;
bool failed = false;
Goals waitees;
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
localOutputInfo && localOutputInfo->outPath != depPath) {
warn(
"substituter '%s' has an incompatible realisation for '%s', ignoring.\n"
"Local: %s\n"
"Remote: %s",
sub->config.getHumanReadableURI(),
depId.to_string(),
worker.store.printStorePath(localOutputInfo->outPath),
worker.store.printStorePath(depPath));
failed = true;
break;
}
waitees.insert(worker.makeDrvOutputSubstitutionGoal(depId));
}
}
if (failed)
continue;
waitees.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
co_await await(std::move(waitees));
trace("output path substituted");
if (nrFailed > 0) {
debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
co_return amDone(nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed);
}
worker.store.registerDrvOutput({*outputInfo, id});
trace("finished");
co_return amDone(ecSuccess);
}
+2 -1
View File
@@ -1,4 +1,5 @@
#include "nix/store/build/worker.hh"
#include "nix/store/store-open.hh"
#include "nix/store/build/substitution-goal.hh"
#include "nix/store/nar-info.hh"
#include "nix/util/finally.hh"
@@ -59,7 +60,7 @@ Goal::Co PathSubstitutionGoal::init()
throw Error(
"cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath));
auto subs = worker.getSubstituters();
auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
bool substituterFailed = false;
std::optional<Error> lastStoresException = std::nullopt;
-2
View File
@@ -1,6 +1,5 @@
#include "nix/store/local-store.hh"
#include "nix/store/machines.hh"
#include "nix/store/store-open.hh"
#include "nix/store/build/worker.hh"
#include "nix/store/build/substitution-goal.hh"
#include "nix/store/build/drv-output-substitution-goal.hh"
@@ -22,7 +21,6 @@ Worker::Worker(Store & store, Store & evalStore)
, actSubstitutions(*logger, actCopyPaths)
, store(store)
, evalStore(evalStore)
, getSubstituters{[] { return settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>{}; }}
{
nrLocalBuilds = 0;
nrSubstitutions = 0;
+1 -5
View File
@@ -341,12 +341,8 @@ DerivationOptions<SingleDerivedPath> derivationOptionsFromStructuredAttrs(
if (parsed) {
auto * e = optionalValueAt(parsed->structuredAttrs, "exportReferencesGraph");
if (!e)
if (!e || !e->is_object())
return ret;
if (!e->is_object()) {
warn("'exportReferencesGraph' in structured attrs is not a JSON object, ignoring");
return ret;
}
for (auto & [key, storePathsJson] : getObject(*e)) {
StringSet ss;
flatten(storePathsJson, ss);
@@ -14,14 +14,11 @@ namespace nix {
class Worker;
/**
* Fetch a `Realisation` (drv ⨯ output name -> output path) from a
* substituter.
*
* If the output store object itself should also be substituted, that is
* the responsibility of the caller to do so.
*
* @todo rename this `BuidlTraceEntryGoal`, which will make sense
* especially once `Realisation` is renamed to `BuildTraceEntry`.
* Substitution of a derivation output.
* This is done in three steps:
* 1. Fetch the output info from a substituter
* 2. Substitute the corresponding output path
* 3. Register the output info
*/
class DrvOutputSubstitutionGoal : public Goal
{
@@ -34,12 +31,6 @@ class DrvOutputSubstitutionGoal : public Goal
public:
DrvOutputSubstitutionGoal(const DrvOutput & id, Worker & worker);
/**
* The realisation corresponding to the given output id.
* Will be filled once we can get it.
*/
std::shared_ptr<const UnkeyedRealisation> outputInfo;
Co init();
std::string key() override;
@@ -8,7 +8,6 @@
#include "nix/store/realisation.hh"
#include "nix/util/muxable-pipe.hh"
#include <functional>
#include <future>
#include <thread>
@@ -172,14 +171,6 @@ public:
Store & store;
Store & evalStore;
/**
* Function to get the substituters to use for path substitution.
*
* Defaults to `getDefaultSubstituters`. This allows tests to
* inject custom substituters.
*/
std::function<std::list<ref<Store>>()> getSubstituters;
#ifndef _WIN32 // TODO Enable building on Windows
std::unique_ptr<HookInstance> hook;
#endif
@@ -33,22 +33,6 @@ private:
public:
PathLocks();
PathLocks(const std::set<std::filesystem::path> & paths, const std::string & waitMsg = "");
PathLocks(PathLocks && other) noexcept
: fds(std::exchange(other.fds, {}))
, deletePaths(other.deletePaths)
{
}
PathLocks & operator=(PathLocks && other) noexcept
{
fds = std::exchange(other.fds, {});
deletePaths = other.deletePaths;
return *this;
}
PathLocks(const PathLocks &) = delete;
PathLocks & operator=(const PathLocks &) = delete;
bool lockPaths(const std::set<std::filesystem::path> & _paths, const std::string & waitMsg = "", bool wait = true);
~PathLocks();
void unlock();
@@ -61,10 +45,6 @@ struct FdLock
bool acquired = false;
FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
FdLock(const FdLock &) = delete;
FdLock & operator=(const FdLock &) = delete;
FdLock(FdLock &&) = delete;
FdLock & operator=(FdLock &&) = delete;
~FdLock()
{
@@ -56,6 +56,14 @@ struct UnkeyedRealisation
StringSet signatures;
/**
* The realisations that are required for the current one to be valid.
*
* When importing this realisation, the store will first check that all its
* dependencies exist, and map to the correct output path
*/
std::map<DrvOutput, StorePath> dependentRealisations;
std::string fingerprint(const DrvOutput & key) const;
void sign(const DrvOutput & key, const Signer &);
@@ -79,6 +87,10 @@ struct Realisation : UnkeyedRealisation
bool isCompatibleWith(const UnkeyedRealisation & other) const;
static std::set<Realisation> closure(Store &, const std::set<Realisation> &);
static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res);
bool operator==(const Realisation &) const = default;
auto operator<=>(const Realisation &) const = default;
};
@@ -142,6 +154,10 @@ struct RealisedPath
*/
const StorePath & path() const &;
void closure(Store & store, Set & ret) const;
static void closure(Store & store, const Set & startPaths, Set & ret);
Set closure(Store & store) const;
bool operator==(const RealisedPath &) const = default;
auto operator<=>(const RealisedPath &) const = default;
};
@@ -1007,6 +1007,9 @@ decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashR
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
std::map<DrvOutput, StorePath>
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore = nullptr);
template<>
struct json_avoids_null<TrustedFlag> : std::true_type
{};
+51
View File
@@ -110,6 +110,8 @@ struct LocalStore::State::Stmts
SQLiteStmt QueryAllRealisedOutputs;
SQLiteStmt QueryPathFromHashPart;
SQLiteStmt QueryValidPaths;
SQLiteStmt QueryRealisationReferences;
SQLiteStmt AddRealisationReference;
};
LocalStore::LocalStore(ref<const Config> config)
@@ -388,6 +390,21 @@ LocalStore::LocalStore(ref<const Config> config)
where drvPath = ?
;
)");
state->stmts->QueryRealisationReferences.create(
state->db,
R"(
select drvPath, outputName from Realisations
join RealisationsRefs on realisationReference = Realisations.id
where referrer = ?;
)");
state->stmts->AddRealisationReference.create(
state->db,
R"(
insert or replace into RealisationsRefs (referrer, realisationReference)
values (
(select id from Realisations where drvPath = ? and outputName = ?),
(select id from Realisations where drvPath = ? and outputName = ?));
)");
}
}
@@ -637,6 +654,25 @@ void LocalStore::registerDrvOutput(const Realisation & info)
concatStringsSep(" ", info.signatures))
.exec();
}
for (auto & [outputId, depPath] : info.dependentRealisations) {
auto localRealisation = queryRealisationCore_(*state, outputId);
if (!localRealisation)
throw Error(
"unable to register the derivation '%s' as it "
"depends on the non existent '%s'",
info.id.to_string(),
outputId.to_string());
if (localRealisation->second.outPath != depPath)
throw Error(
"unable to register the derivation '%s' as it "
"depends on a realisation of '%s' that doesn’t"
"match what we have locally",
info.id.to_string(),
outputId.to_string());
state->stmts->AddRealisationReference
.use()(info.id.strHash())(info.id.outputName)(outputId.strHash())(outputId.outputName)
.exec();
}
});
}
@@ -1570,6 +1606,21 @@ std::optional<const UnkeyedRealisation> LocalStore::queryRealisation_(LocalStore
return std::nullopt;
auto [realisationDbId, res] = *maybeCore;
std::map<DrvOutput, StorePath> dependentRealisations;
auto useRealisationRefs(state.stmts->QueryRealisationReferences.use()(realisationDbId));
while (useRealisationRefs.next()) {
auto depId = DrvOutput{
Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)),
useRealisationRefs.getStr(1),
};
auto dependentRealisation = queryRealisationCore_(state, depId);
assert(dependentRealisation); // Enforced by the db schema
auto outputPath = dependentRealisation->second.outPath;
dependentRealisations.insert({depId, outputPath});
}
res.dependentRealisations = dependentRealisations;
return {res};
}
+59
View File
@@ -334,6 +334,65 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
result);
}
std::map<DrvOutput, StorePath>
drvOutputReferences(const std::set<Realisation> & inputRealisations, const StorePathSet & pathReferences)
{
std::map<DrvOutput, StorePath> res;
for (const auto & input : inputRealisations) {
if (pathReferences.count(input.outPath)) {
res.insert({input.id, input.outPath});
}
}
return res;
}
std::map<DrvOutput, StorePath>
drvOutputReferences(Store & store, const Derivation & drv, const StorePath & outputPath, Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
std::set<Realisation> inputRealisations;
auto accumRealisations = [&](this auto & self,
const StorePath & inputDrv,
const DerivedPathMap<StringSet>::ChildNode & inputNode) -> void {
if (!inputNode.value.empty()) {
auto outputHashes = staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv));
for (const auto & outputName : inputNode.value) {
auto outputHash = get(outputHashes, outputName);
if (!outputHash)
throw Error(
"output '%s' of derivation '%s' isn't realised", outputName, store.printStorePath(inputDrv));
DrvOutput key{*outputHash, outputName};
auto thisRealisation = store.queryRealisation(key);
if (!thisRealisation)
throw Error(
"output '%s' of derivation '%s' isn’t built", outputName, store.printStorePath(inputDrv));
inputRealisations.insert({*thisRealisation, std::move(key)});
}
}
if (!inputNode.value.empty()) {
auto d = makeConstantStorePathRef(inputDrv);
for (const auto & [outputName, childNode] : inputNode.childMap) {
SingleDerivedPath next = SingleDerivedPath::Built{d, outputName};
self(
// TODO deep resolutions for dynamic derivations, issue #8947, would go here.
resolveDerivedPath(store, next, evalStore_),
childNode);
}
}
};
for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map)
accumRealisations(inputDrv, inputNode);
auto info = store.queryPathInfo(outputPath);
return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references);
}
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
+83 -3
View File
@@ -1,5 +1,6 @@
#include "nix/store/realisation.hh"
#include "nix/store/store-api.hh"
#include "nix/util/closure.hh"
#include "nix/util/signature/local-keys.hh"
#include "nix/util/json-utils.hh"
#include <nlohmann/json.hpp>
@@ -25,6 +26,41 @@ std::string DrvOutput::to_string() const
return strHash() + "!" + outputName;
}
std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs)
{
std::set<Realisation> res;
Realisation::closure(store, startOutputs, res);
return res;
}
void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res)
{
auto getDeps = [&](const Realisation & current) -> std::set<Realisation> {
std::set<Realisation> res;
for (auto & [currentDep, _] : current.dependentRealisations) {
if (auto currentRealisation = store.queryRealisation(currentDep))
res.insert({*currentRealisation, currentDep});
else
throw Error("Unrealised derivation '%s'", currentDep.to_string());
}
return res;
};
computeClosure<Realisation>(
startOutputs,
res,
[&](const Realisation & current, std::function<void(std::promise<std::set<Realisation>> &)> processEdges) {
std::promise<std::set<Realisation>> promise;
try {
auto res = getDeps(current);
promise.set_value(res);
} catch (...) {
promise.set_exception(std::current_exception());
}
return processEdges(promise);
});
}
std::string UnkeyedRealisation::fingerprint(const DrvOutput & key) const
{
nlohmann::json serialized = Realisation{*this, key};
@@ -63,7 +99,43 @@ const StorePath & RealisedPath::path() const &
bool Realisation::isCompatibleWith(const UnkeyedRealisation & other) const
{
return outPath == other.outPath;
if (outPath == other.outPath) {
if (dependentRealisations.empty() != other.dependentRealisations.empty()) {
warn(
"Encountered a realisation for '%s' with an empty set of "
"dependencies. This is likely an artifact from an older Nix. "
"I’ll try to fix the realisation if I can",
id.to_string());
return true;
} else if (dependentRealisations == other.dependentRealisations) {
return true;
}
}
return false;
}
void RealisedPath::closure(Store & store, const RealisedPath::Set & startPaths, RealisedPath::Set & ret)
{
// FIXME: This only builds the store-path closure, not the real realisation
// closure
StorePathSet initialStorePaths, pathsClosure;
for (auto & path : startPaths)
initialStorePaths.insert(path.path());
store.computeFSClosure(initialStorePaths, pathsClosure);
ret.insert(startPaths.begin(), startPaths.end());
ret.insert(pathsClosure.begin(), pathsClosure.end());
}
void RealisedPath::closure(Store & store, RealisedPath::Set & ret) const
{
RealisedPath::closure(store, {*this}, ret);
}
RealisedPath::Set RealisedPath::closure(Store & store) const
{
RealisedPath::Set ret;
closure(store, ret);
return ret;
}
} // namespace nix
@@ -90,19 +162,27 @@ UnkeyedRealisation adl_serializer<UnkeyedRealisation>::from_json(const json & js
if (auto signaturesOpt = optionalValueAt(json, "signatures"))
signatures = *signaturesOpt;
std::map<DrvOutput, StorePath> dependentRealisations;
if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations"))
for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies))
dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath});
return UnkeyedRealisation{
.outPath = valueAt(json, "outPath"),
.signatures = signatures,
.dependentRealisations = dependentRealisations,
};
}
void adl_serializer<UnkeyedRealisation>::to_json(json & json, const UnkeyedRealisation & r)
{
auto jsonDependentRealisations = nlohmann::json::object();
for (auto & [depId, depOutPath] : r.dependentRealisations)
jsonDependentRealisations.emplace(depId.to_string(), depOutPath);
json = {
{"outPath", r.outPath},
{"signatures", r.signatures},
// back-compat
{"dependentRealisations", json::object()},
{"dependentRealisations", jsonDependentRealisations},
};
}
+1 -1
View File
@@ -292,7 +292,7 @@ std::vector<KeyedBuildResult> RestrictedStore::buildPathsWithResults(
next->computeFSClosure(newPaths, closure);
for (auto & path : closure)
goal.addDependency(path);
for (auto & real : newRealisations)
for (auto & real : Realisation::closure(*next, newRealisations))
goal.addedDrvOutputs.insert(real.id);
return results;
+22 -18
View File
@@ -915,21 +915,36 @@ std::map<StorePath, StorePath> copyPaths(
SubstituteFlag substitute)
{
StorePathSet storePaths;
std::vector<const Realisation *> realisations;
std::set<Realisation> toplevelRealisations;
for (auto & path : paths) {
storePaths.insert(path.path());
if (auto * realisation = std::get_if<Realisation>(&path.raw)) {
experimentalFeatureSettings.require(Xp::CaDerivations);
realisations.push_back(realisation);
toplevelRealisations.insert(*realisation);
}
}
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
try {
// Copy the realisations. TODO batch this
for (const auto * realisation : realisations)
dstStore.registerDrvOutput(*realisation, checkSigs);
// Copy the realisation closure
processGraph<Realisation>(
Realisation::closure(srcStore, toplevelRealisations),
[&](const Realisation & current) -> std::set<Realisation> {
std::set<Realisation> children;
for (const auto & [drvOutput, _] : current.dependentRealisations) {
auto currentChild = srcStore.queryRealisation(drvOutput);
if (!currentChild)
throw Error(
"incomplete realisation closure: '%s' is a "
"dependency of '%s' but isn't registered",
drvOutput.to_string(),
current.id.to_string());
children.insert({*currentChild, drvOutput});
}
return children;
},
[&](const Realisation & current) -> void { dstStore.registerDrvOutput(current, checkSigs); });
} catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want
@@ -1040,19 +1055,8 @@ void copyClosure(
if (&srcStore == &dstStore)
return;
StorePathSet closure0;
for (auto & path : paths) {
if (auto * opaquePath = std::get_if<OpaquePath>(&path.raw)) {
closure0.insert(opaquePath->path);
}
}
StorePathSet closure1;
srcStore.computeFSClosure(closure0, closure1);
RealisedPath::Set closure = paths;
for (auto && path : closure1)
closure.insert({std::move(path)});
RealisedPath::Set closure;
RealisedPath::closure(srcStore, paths, closure);
copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
}
@@ -83,31 +83,6 @@ void checkpointJson(CharacterizationTest & test, PathView testStem, const T & go
}
}
/**
* Specialization for when we need to do "JSON -> `ref<T>`" in one
* direction, but "`const T &` -> JSON" in the other direction.
*/
template<typename T>
void checkpointJson(CharacterizationTest & test, PathView testStem, const ref<T> & got)
{
using namespace nlohmann;
auto file = test.goldenMaster(Path{testStem} + ".json");
json gotJson = static_cast<json>(*got);
if (testAccept()) {
std::filesystem::create_directories(file.parent_path());
writeFile(file, gotJson.dump(2) + "\n");
ADD_FAILURE() << "Updating golden master " << file;
} else {
json expectedJson = json::parse(readFile(file));
ASSERT_EQ(gotJson, expectedJson);
ref<T> expected = adl_serializer<ref<T>>::from_json(expectedJson);
ASSERT_EQ(*got, *expected);
}
}
/**
* Mixin class for writing characterization tests for `nlohmann::json`
* conversions for a given type.
+1 -1
View File
@@ -82,7 +82,7 @@ protected:
void TearDown() override
{
delTmpDir.reset();
delTmpDir.release();
}
};
+2 -1
View File
@@ -103,7 +103,8 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
{
auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true);
SourcePath path2 = {
makeFSSourceAccessor(std::filesystem::path{}, /*trackLastModified=*/true), CanonPath(absPath(path))};
path2.dumpPath(sink, filter);
return path2.accessor->getLastModified().value();
}
-11
View File
@@ -54,17 +54,6 @@ std::optional<CanonPath> CanonPath::parent() const
return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
}
std::optional<std::pair<CanonPath, std::string_view>> CanonPath::parentAndChild() const
{
if (isRoot())
return std::nullopt;
auto slash = path.rfind('/');
return {{
CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, slash))),
std::string_view{path}.substr(slash + 1)
}};
}
void CanonPath::pop()
{
assert(!isRoot());
+14 -77
View File
@@ -69,54 +69,24 @@ struct ArchiveDecompressionSource : Source
}
};
/* These strings are a part of the public API in store parameters and such. Do not change!
Happens to match enum names. */
#define NIX_FOR_EACH_LA_ALGO(MACRO) \
MACRO(bzip2) \
MACRO(compress) \
MACRO(grzip) \
MACRO(gzip) \
MACRO(lrzip) \
MACRO(lz4) \
MACRO(lzip) \
MACRO(lzma) \
MACRO(lzop) \
MACRO(xz) \
MACRO(zstd)
struct ArchiveCompressionSink : CompressionSink
{
Sink & nextSink;
struct archive * archive;
ArchiveCompressionSink(
Sink & nextSink, CompressionAlgo method, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT)
: nextSink(nextSink)
{
archive = archive_write_new();
if (!archive)
throw Error("failed to initialize libarchive");
auto [addFilter, format] = [method]() -> std::pair<int (*)(struct archive *), const char *> {
switch (method) {
case CompressionAlgo::none:
case CompressionAlgo::brotli:
unreachable();
#define NIX_DEF_LA_ALGO_CASE(algo) \
case CompressionAlgo::algo: \
return {archive_write_add_filter_##algo, #algo};
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
#undef NIX_DEF_LA_ALGO_CASE
}
unreachable();
}();
check(addFilter(archive), "couldn't initialize compression (%s)");
check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
check(archive_write_set_format_raw(archive));
if (parallel)
check(archive_write_set_filter_option(archive, format, "threads", "0"));
check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
if (level != COMPRESSION_LEVEL_DEFAULT)
check(archive_write_set_filter_option(archive, format, "compression-level", std::to_string(level).c_str()));
check(archive_write_set_filter_option(
archive, format.c_str(), "compression-level", std::to_string(level).c_str()));
// disable internal buffering
check(archive_write_set_bytes_per_block(archive, 0));
// disable output padding
@@ -319,52 +289,19 @@ struct BrotliCompressionSink : ChunkedCompressionSink
}
};
/* Parses a *compression* method into the corresponding enum. This is only used
in the *compression* case and user interface. Content-Encoding should not use
these. */
static CompressionAlgo parseNixCompressionAlgoString(std::string_view method)
{
static const std::unordered_map<std::string_view, CompressionAlgo> lookupTable = {
{"none", CompressionAlgo::none},
{"br", CompressionAlgo::brotli},
#define NIX_DEF_LA_ALGO_NAME(algo) {#algo, CompressionAlgo::algo},
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_NAME)
#undef NIX_DEF_LA_ALGO_NAME
};
if (auto it = lookupTable.find(method); it != lookupTable.end())
return it->second;
static const StringSet allNames = [&]() {
StringSet res;
for (auto & [name, _] : lookupTable)
res.emplace(name);
return res;
}();
throw UnknownCompressionMethod(
Suggestions::bestMatches(allNames, method), "unknown compression method '%s'", method);
}
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level)
{
return makeCompressionSink(parseNixCompressionAlgoString(method), nextSink, parallel, level);
}
ref<CompressionSink> makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel, int level)
{
switch (method) {
case CompressionAlgo::none:
return make_ref<NoneSink>(nextSink);
case CompressionAlgo::brotli:
return make_ref<BrotliCompressionSink>(nextSink);
/* Everything else is supported via libarchive. */
#define NIX_DEF_LA_ALGO_CASE(algo) case CompressionAlgo::algo:
NIX_FOR_EACH_LA_ALGO(NIX_DEF_LA_ALGO_CASE)
std::vector<std::string> la_supports = {
"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"};
if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level);
#undef NIX_DEF_LA_ALGO_CASE
}
unreachable();
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "br")
return make_ref<BrotliCompressionSink>(nextSink);
else
throw UnknownCompressionMethod("unknown compression method '%s'", method);
}
std::string compress(const std::string & method, std::string_view in, const bool parallel, int level)
@@ -11,24 +11,6 @@ class AutoRemoveJail
bool del;
public:
AutoRemoveJail(int jid);
AutoRemoveJail(const AutoRemoveJail &) = delete;
AutoRemoveJail & operator=(const AutoRemoveJail &) = delete;
AutoRemoveJail(AutoRemoveJail && other) noexcept
: jid(other.jid)
, del(other.del)
{
other.cancel();
}
AutoRemoveJail & operator=(AutoRemoveJail && other) noexcept
{
jid = other.jid;
del = other.del;
other.cancel();
return *this;
}
AutoRemoveJail();
~AutoRemoveJail();
void cancel();
+26 -15
View File
@@ -70,7 +70,31 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
return dst;
}
void RestoreSink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
#ifndef _WIN32
void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
{
if (path.isRoot()) {
createDirectory(path);
callback(*this, path);
return;
}
createDirectory(path);
assert(dirFd); // If that's not true the above call must have thrown an exception.
RestoreSink dirSink{startFsync};
dirSink.dstPath = append(dstPath, path);
dirSink.dirFd =
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirSink.dirFd)
throw SysError("opening directory '%s'", dirSink.dstPath.string());
callback(dirSink, CanonPath::root);
}
#endif
void RestoreSink::createDirectory(const CanonPath & path)
{
auto p = append(dstPath, path);
@@ -83,16 +107,6 @@ void RestoreSink::createDirectory(const CanonPath & path, std::optional<Director
if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
throw SysError("creating directory '%s'", p.string());
if (callback) {
RestoreSink dirSink{startFsync};
dirSink.dstPath =
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirSink.dirFd)
throw SysError("opening directory '%s'", dirSink.dstPath.string());
(*callback)(dirSink, CanonPath::root);
}
return;
}
#endif
@@ -111,10 +125,7 @@ void RestoreSink::createDirectory(const CanonPath & path, std::optional<Director
throw SysError("creating directory '%1%'", p.string());
}
#endif
if (callback)
(*callback)(*this, path);
}
};
struct RestoreRegularFile : CreateRegularFileSink
{
@@ -238,13 +238,6 @@ public:
return ((std::string_view) path).substr(path.rfind('/') + 1);
}
/**
* Combination of `parent` and `baseName`.
*
* @note that because of the `std::string_view`, we are borrowing for the base name.
*/
std::optional<std::pair<CanonPath, std::string_view>> parentAndChild() const;
bool operator==(const CanonPath & x) const
{
return path == x.path;
@@ -16,22 +16,6 @@ struct CompressionSink : BufferedSink, FinishSink
using FinishSink::finish;
};
enum class CompressionAlgo {
none,
brotli,
bzip2,
compress,
grzip,
gzip,
lrzip,
lz4,
lzip,
lzma,
lzop,
xz,
zstd,
};
std::string decompress(const std::string & method, std::string_view in);
std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
@@ -41,9 +25,6 @@ std::string compress(const std::string & method, std::string_view in, const bool
ref<CompressionSink>
makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1);
ref<CompressionSink>
makeCompressionSink(CompressionAlgo method, Sink & nextSink, const bool parallel = false, int level = -1);
MakeError(UnknownCompressionMethod, Error);
MakeError(CompressionError, Error);
+1 -16
View File
@@ -480,23 +480,8 @@ class AutoUnmount
Path path;
bool del;
public:
AutoUnmount();
AutoUnmount(Path &);
AutoUnmount(const AutoUnmount &) = delete;
AutoUnmount(AutoUnmount && other) noexcept
: path(std::move(other.path))
, del(std::exchange(other.del, false))
{
}
AutoUnmount & operator=(AutoUnmount && other) noexcept
{
path = std::move(other.path);
del = std::exchange(other.del, false);
return *this;
}
AutoUnmount();
~AutoUnmount();
void cancel();
};
+14 -14
View File
@@ -34,6 +34,8 @@ struct FileSystemObjectSink
{
virtual ~FileSystemObjectSink() = default;
virtual void createDirectory(const CanonPath & path) = 0;
using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
/**
@@ -45,7 +47,11 @@ struct FileSystemObjectSink
* freshly created directory. Use this when it's important to disallow any
* intermediate path components from being symlinks.
*/
virtual void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) = 0;
virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
{
createDirectory(path);
callback(*this, path);
}
/**
* This function in general is no re-entrant. Only one file can be
@@ -80,11 +86,7 @@ void copyRecursive(
*/
struct NullFileSystemObjectSink : FileSystemObjectSink
{
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
{
if (callback)
(*callback)(*this, CanonPath::root);
}
void createDirectory(const CanonPath & path) override {}
void createSymlink(const CanonPath & path, const std::string & target) override {}
@@ -116,7 +118,11 @@ struct RestoreSink : FileSystemObjectSink
{
}
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
void createDirectory(const CanonPath & path) override;
#ifndef _WIN32
void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
#endif
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
@@ -138,15 +144,9 @@ struct RegularFileSink : FileSystemObjectSink
{
}
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
void createDirectory(const CanonPath & path) override
{
regular = false;
if (callback) {
NullFileSystemObjectSink s;
/* The children of the
directory that cannot exist also cannot exist */
(*callback)(s, CanonPath::root);
}
}
void createSymlink(const CanonPath & path, const std::string & target) override
@@ -153,7 +153,7 @@ struct MemorySink : FileSystemObjectSink
{
}
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
void createDirectory(const CanonPath & path) override;
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
@@ -43,36 +43,6 @@ public:
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
/**
* Create a `PosixSourceAccessor` and `SourcePath` corresponding to
* some native path.
*
* @param Whether the accessor should return a non-null getLastModified.
* When true the accessor must be used only by a single thread.
*
* The `PosixSourceAccessor` is rooted as far up the tree as
* possible, (e.g. on Windows it could scoped to a drive like
* `C:\`). This allows more `..` parent accessing to work.
*
* @note When `path` is trusted user input, canonicalize it using
* `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc,
* as appropriate for the use case. At least weak canonicalization is
* required for the `SourcePath` to do anything useful at the location it
* points to.
*
* @note A canonicalizing behavior is not built in `createAtRoot` so that
* callers do not accidentally introduce symlink-related security vulnerabilities.
* Furthermore, `createAtRoot` does not know whether the file pointed to by
* `path` should be resolved if it is itself a symlink. In other words,
* `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers.
*
* See
* [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path)
* and
* [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path).
*/
static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false);
std::optional<std::time_t> getLastModified() override
{
return trackLastModified ? std::optional{mtime} : std::nullopt;
+1 -20
View File
@@ -159,8 +159,6 @@ struct FdSink : BufferedSink
}
FdSink(FdSink &&) = default;
FdSink(const FdSink &) = delete;
FdSink & operator=(const FdSink &) = delete;
FdSink & operator=(FdSink && s)
{
@@ -202,10 +200,8 @@ struct FdSource : BufferedSource, RestartableSource
}
FdSource(FdSource &&) = default;
FdSource & operator=(FdSource && s) = default;
FdSource(const FdSource &) = delete;
FdSource & operator=(const FdSource & s) = delete;
~FdSource() = default;
bool good() override;
void restart() override;
@@ -456,11 +452,6 @@ struct LambdaSink : Sink
{
}
LambdaSink(LambdaSink &&) = delete;
LambdaSink(const LambdaSink &) = delete;
LambdaSink & operator=(LambdaSink &&) = delete;
LambdaSink & operator=(const LambdaSink &) = delete;
~LambdaSink()
{
cleanupFun();
@@ -637,11 +628,6 @@ struct FramedSource : Source
{
}
FramedSource(FramedSource &&) = delete;
FramedSource(const FramedSource &) = delete;
FramedSource & operator=(FramedSource &&) = delete;
FramedSource & operator=(const FramedSource &) = delete;
~FramedSource()
{
try {
@@ -699,11 +685,6 @@ struct FramedSink : nix::BufferedSink
{
}
FramedSink(FramedSink &&) = delete;
FramedSink(const FramedSink &) = delete;
FramedSink & operator=(FramedSink &&) = delete;
FramedSink & operator=(const FramedSink &) = delete;
~FramedSink()
{
try {
+6 -7
View File
@@ -69,10 +69,13 @@ public:
{
}
public:
Lock(Lock && l) = delete;
Lock(Lock && l)
: s(l.s)
{
unreachable();
}
Lock(const Lock & l) = delete;
Lock & operator=(Lock && l) = delete;
Lock & operator=(const Lock & l) = delete;
~Lock() {}
@@ -107,8 +110,6 @@ public:
struct WriteLock : Lock<WL>
{
using Lock<WL>::Lock;
T * operator->()
{
return &WriteLock::s->data;
@@ -130,8 +131,6 @@ public:
struct ReadLock : Lock<RL>
{
using Lock<RL>::Lock;
const T * operator->()
{
return &ReadLock::s->data;
-5
View File
@@ -384,11 +384,6 @@ struct MaintainCount
counter += delta;
}
MaintainCount(MaintainCount &&) = delete;
MaintainCount(const MaintainCount &) = delete;
MaintainCount & operator=(MaintainCount &&) = delete;
MaintainCount & operator=(const MaintainCount &) = delete;
~MaintainCount()
{
counter -= delta;
@@ -50,11 +50,6 @@ public:
writer.openElement(name, attrs);
}
XMLOpenElement(XMLOpenElement &&) = delete;
XMLOpenElement(const XMLOpenElement &) = delete;
XMLOpenElement & operator=(XMLOpenElement &&) = delete;
XMLOpenElement & operator=(const XMLOpenElement &) = delete;
~XMLOpenElement()
{
writer.closeElement();
+2 -5
View File
@@ -146,7 +146,7 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents
using File = MemorySourceAccessor::File;
void MemorySink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
void MemorySink::createDirectory(const CanonPath & path)
{
auto * f = dst.open(path, File{File::Directory{}});
if (!f)
@@ -154,10 +154,7 @@ void MemorySink::createDirectory(const CanonPath & path, std::optional<Directory
if (!std::holds_alternative<File::Directory>(f->raw))
throw Error("file '%s' is not a directory", path);
if (callback)
(*callback)(*this, path);
}
};
struct CreateMemoryRegularFile : CreateRegularFileSink
{
+1 -3
View File
@@ -98,13 +98,11 @@ struct NarAccessor : public SourceAccessor
}
}
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
void createDirectory(const CanonPath & path) override
{
createMember(
path,
NarMember{.stat = {.type = Type::tDirectory, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
if (callback)
(*callback)(*this, path);
}
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
-11
View File
@@ -1,7 +1,5 @@
#include "nix/util/posix-source-accessor.hh"
#include "nix/util/source-path.hh"
#include "nix/util/signals.hh"
#include "nix/util/sync.hh"
#include <boost/unordered/concurrent_flat_map.hpp>
@@ -20,15 +18,6 @@ PosixSourceAccessor::PosixSourceAccessor()
{
}
SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified)
{
std::filesystem::path path2 = absPath(path);
return {
make_ref<PosixSourceAccessor>(path2.root_path(), trackLastModified),
CanonPath{path2.relative_path().string()},
};
}
std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path)
{
return root.empty() ? (std::filesystem::path{path.abs()})
@@ -27,10 +27,6 @@ private:
public:
MonitorFdHup(int fd);
MonitorFdHup(MonitorFdHup &&) = delete;
MonitorFdHup(const MonitorFdHup &) = delete;
MonitorFdHup & operator=(MonitorFdHup &&) = delete;
MonitorFdHup & operator=(const MonitorFdHup &) = delete;
~MonitorFdHup()
{
+5 -11
View File
@@ -141,16 +141,6 @@ struct InterruptCallbackImpl : InterruptCallback
{
InterruptCallbacks::Token token;
InterruptCallbackImpl(InterruptCallbacks::Token token)
: token(token)
{
}
InterruptCallbackImpl(InterruptCallbackImpl &&) = delete;
InterruptCallbackImpl(const InterruptCallbackImpl &) = delete;
InterruptCallbackImpl & operator=(InterruptCallbackImpl &&) = delete;
InterruptCallbackImpl & operator=(const InterruptCallbackImpl &) = delete;
~InterruptCallbackImpl() override
{
auto interruptCallbacks(_interruptCallbacks.lock());
@@ -163,7 +153,11 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()>
auto interruptCallbacks(_interruptCallbacks.lock());
auto token = interruptCallbacks->nextToken++;
interruptCallbacks->callbacks.emplace(token, callback);
return std::make_unique<InterruptCallbackImpl>(token);
std::unique_ptr<InterruptCallbackImpl> res{new InterruptCallbackImpl{}};
res->token = token;
return std::unique_ptr<InterruptCallback>(res.release());
}
} // namespace nix
+1 -1
View File
@@ -38,7 +38,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
if (!namePart)
namePart = baseNameOf(path);
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
SourcePath sourcePath = makeFSSourceAccessor(makeParentCanonical(path));
auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first
: store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path;
+1 -3
View File
@@ -85,9 +85,7 @@ struct CmdHashBase : Command
return std::make_unique<HashSink>(hashAlgo);
};
auto makeSourcePath = [&]() -> SourcePath {
return PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
};
auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(makeParentCanonical(path)); };
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
switch (mode) {
+2 -2
View File
@@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
throw UsageError("unknown flag");
for (auto & i : opArgs) {
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
SourcePath sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath)));
}
}
@@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
opArgs.pop_front();
for (auto & i : opArgs) {
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
SourcePath sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
std::cout << fmt(
"%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path));
}
+2 -2
View File
@@ -256,7 +256,7 @@ hashPath(char * algo, int base32, char * path)
PPCODE:
try {
Hash h = hashPath(
PosixSourceAccessor::createAtRoot(path),
SourcePath{getFSSourceAccessor(), CanonPath(absPath(Path(path)))},
FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first;
auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
@@ -336,7 +336,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo)
auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat;
auto path = THIS->store->addToStore(
std::string(baseNameOf(srcPath)),
PosixSourceAccessor::createAtRoot(srcPath),
SourcePath{getFSSourceAccessor(), CanonPath(absPath(Path(srcPath)))},
method, parseHashAlgo(algo));
XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0)));
} catch (Error & e) {
@@ -25,9 +25,4 @@ nix build -f nondeterministic.nix dep2 --no-link
# If everything goes right, we should rebuild dep2 rather than fetch it from
# the cache (because that would mean duplicating `current-time` in the closure),
# and have `dep1 == dep2`.
# FIXME: Force the use of small-step resolutions only to fix this in a
# better way (#11896, #11928).
skipTest "temporarily broken because dependent realisations are removed"
nix build --substituters "$REMOTE_STORE" -f nondeterministic.nix toplevel --no-require-sigs --no-link
+1 -4
View File
@@ -22,10 +22,7 @@ nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix
# Restart the build on an empty store, ensuring that we don't build
clearStore
# FIXME: `dependentCA` should not need to be explicitly mentioned in
# this. Force the use of small-step resolutions only to allow not
# mentioning it explicitly again. (#11896, #11928).
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA dependentCA
buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA
# Check that the thing we’ve just substituted has its realisation stored
nix realisation info --file ./content-addressed.nix transitivelyDependentCA
# Check that its dependencies have it too
-13
View File
@@ -50,16 +50,3 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet "
# Check it works with the expected structured attrs
hacky=$(nix-instantiate --expr "$hackyExpr")
nix derivation show "$hacky" | jq --exit-status '.derivations."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}'
# Test warning for non-object exportReferencesGraph in structured attrs
# shellcheck disable=SC2016
expectStderr 0 nix-build --expr '
with import ./config.nix;
mkDerivation {
name = "export-graph-non-object";
__structuredAttrs = true;
exportReferencesGraph = [ "foo" "bar" ];
builder = "/bin/sh";
args = ["-c" "echo foo > ${builtins.placeholder "out"}"];
}
' | grepQuiet "warning:.*exportReferencesGraph.*not a JSON object"