Compare commits

..
Author SHA1 Message Date
John Ericson fdfc8bf554 Make the git sink support bare files 2025-12-17 17:06:29 -05:00
John Ericson 4586223bab GitFileSystemObjectSink: Avoid the name in the root builder
It is not actually used, and I rather not keep such "fictitious value"
around.
2025-12-17 17:02:33 -05:00
John Ericson a53317347d Deduplication in GitFileSystemObjectSink
A bunch of callers to `prepareDirs` were doing the same thing again and
again.
2025-12-17 17:01:39 -05:00
John Ericson 102c0bd9e3 Simplify Git FSO sink
`prepareDirs` always returns true.
2025-12-17 17:01:39 -05:00
John Ericson 1adc04f421 Combine the FileSystemObjectSink::createDirectory methods
I find the implementation easier to understand. And it enables the
refactors that come next.
2025-12-17 17:01:38 -05:00
38 changed files with 730 additions and 909 deletions

No files matched your search

+3 -6
View File
@@ -26,6 +26,7 @@ bash = find_program('bash', native : true)
# HTML manual dependencies (conditional)
if get_option('html-manual')
mdbook = find_program('mdbook', native : true)
rsync = find_program('rsync', required : true, native : true)
endif
pymod = import('python')
@@ -125,12 +126,7 @@ if get_option('html-manual')
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
# Copy source to build directory, excluding the build directory itself
# (which is present when built as an individual component).
# Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests).
(cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -)
chmod -R u+w @2@
find @2@ -name '*.drv' -delete
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
rm -rf @2@/manual
mv @2@/html @2@/manual
@@ -142,6 +138,7 @@ if get_option('html-manual')
mdbook.full_path(),
meson.current_build_dir(),
meson.project_version(),
rsync.full_path(),
),
],
input : [
+2 -2
View File
@@ -10,6 +10,7 @@
mdbook,
jq,
python3,
rsync,
nix-cli,
changelog-d,
json-schema-for-humans,
@@ -53,8 +54,6 @@ mkMesonDerivation (finalAttrs: {
../../src/libstore-tests/data/nar-info
../../src/libstore-tests/data/build-result
../../src/libstore-tests/data/dummy-store
# For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/
../../tests/functional/derivation
# Too many different types of files to filter for now
../../doc/manual
./.
@@ -91,6 +90,7 @@ mkMesonDerivation (finalAttrs: {
]
++ lib.optionals buildHtmlManual [
mdbook
rsync
json-schema-for-humans
]
++ lib.optionals (!officialRelease && buildHtmlManual) [
+39 -52
View File
@@ -148,15 +148,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
isInternal =
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
activeComponentNames = lib.listToAttrs (
map (c: {
name = c.pname or c.name;
value = null;
}) activeComponents
);
isActiveComponent = name: activeComponentNames ? ${name};
in
{
pname = "shell-for-nix";
@@ -199,19 +190,27 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
}
);
small = finalAttrs.finalPackage.withActiveComponents (
c:
lib.intersectAttrs (lib.genAttrs [
"nix-cli"
"nix-util-tests"
"nix-store-tests"
"nix-expr-tests"
"nix-fetchers-tests"
"nix-flake-tests"
"nix-functional-tests"
"nix-perl-bindings"
] (_: null)) c
);
small =
(finalAttrs.finalPackage.withActiveComponents (
c:
lib.intersectAttrs (lib.genAttrs [
"nix-cli"
"nix-util-tests"
"nix-store-tests"
"nix-expr-tests"
"nix-fetchers-tests"
"nix-flake-tests"
"nix-functional-tests"
"nix-perl-bindings"
] (_: null)) c
)).overrideAttrs
(o: {
mesonFlags = o.mesonFlags ++ [
# TODO: infer from activeComponents or vice versa
"-Dkaitai-struct-checks=false"
"-Djson-schema-checks=false"
];
});
};
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
@@ -279,33 +278,21 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
dontUseCmakeConfigure = true;
mesonFlags = [
(lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks"))
(lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
]
++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
++ lib.optionals havePerl (
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
)
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
mesonFlags =
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
++ lib.optionals havePerl (
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
)
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
nativeBuildInputs =
let
inputs =
dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (
lib.lists.concatMap (
# Nix manual has a build-time dependency on nix, but we
# don't want to do a native build just to enter the ross
# dev shell.
#
# TODO: think of a more principled fix for this.
c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs
) activeComponents
)
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
)
++ lib.optional (
!buildCanExecuteHost
@@ -321,8 +308,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
pkgs.buildPackages.nixfmt-rfc-style
pkgs.buildPackages.shellcheck
pkgs.buildPackages.include-what-you-use
pkgs.buildPackages.gdb
]
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
lib.hiPrio pkgs.buildPackages.clang-tools
)
@@ -338,13 +325,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
)
);
buildInputs =
# TODO change Nixpkgs to mark gbenchmark as building on Windows
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
++ lib.optional havePerl pkgs.perl;
buildInputs = [
pkgs.gbenchmark
]
++ dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
)
++ lib.optional havePerl pkgs.perl;
propagatedBuildInputs = dedupByString (v: "${v}") (
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
+46 -7
View File
@@ -70,12 +70,6 @@ TEST_F(GitUtilsTest, sink_basic)
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
// TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory
// here
// The sync method does not document this behavior, should probably renamed because it's not very
// general, and I can't imagine that "non-conventional" archives or any other source to be handled by
// this sink.
sink->createDirectory(CanonPath("foo-1.1"));
sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) {
@@ -91,7 +85,7 @@ TEST_F(GitUtilsTest, sink_basic)
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
auto result = repo->dereferenceSingletonDirectory(sink->flush());
auto result = repo->dereferenceSingletonDirectory(sink->flush().hash);
auto accessor = repo->getAccessor(result, {}, getRepoName());
auto entries = accessor->readDirectory(CanonPath::root);
ASSERT_EQ(entries.size(), 5u);
@@ -123,6 +117,51 @@ TEST_F(GitUtilsTest, sink_hardlink)
}
};
TEST_F(GitUtilsTest, sink_root_file)
{
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
sink->createRegularFile(
CanonPath::root, [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
auto result = sink->flush();
ASSERT_EQ(result.mode, nix::git::Mode::Regular);
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
ASSERT_EQ(accessor->readFile(CanonPath::root), "hello world");
};
TEST_F(GitUtilsTest, sink_root_symlink)
{
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
sink->createSymlink(CanonPath::root, "target");
auto result = sink->flush();
ASSERT_EQ(result.mode, nix::git::Mode::Symlink);
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
ASSERT_EQ(accessor->readLink(CanonPath::root), "target");
};
TEST_F(GitUtilsTest, sink_hardlink_to_root)
{
auto repo = openRepo();
auto sink = repo->getFileSystemObjectSink();
sink->createRegularFile(
CanonPath("hello"), [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
sink->createDirectory(CanonPath("subdir"));
sink->createHardlink(CanonPath("subdir/link"), CanonPath("hello"));
auto result = sink->flush();
ASSERT_EQ(result.mode, nix::git::Mode::Directory);
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
ASSERT_EQ(accessor->readFile(CanonPath("subdir/link")), "hello world");
};
TEST_F(GitUtilsTest, peel_reference)
{
// Create a commit in the repo
+192 -99
View File
@@ -962,15 +962,15 @@ struct GitSourceAccessor : SourceAccessor
Blob getBlob(State & state, const CanonPath & path, bool expectSymlink)
{
if (!expectSymlink && git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
return dupObject<Blob>((git_blob *) &*state.root);
auto notExpected = [&]() {
throw Error(expectSymlink ? "'%s' is not a symlink" : "'%s' is not a regular file", showPath(path));
};
if (path.isRoot())
if (path.isRoot()) {
if (git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
return dupObject<Blob>((git_blob *) &*state.root);
notExpected();
}
auto entry = need(state, path);
@@ -1054,16 +1054,24 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
}
};
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
ref<GitRepoImpl> repo;
namespace {
struct DirEntry
{
git_oid oid;
git_filemode_t mode;
};
struct DirectoryState
{
struct PendingDir
{
std::string name;
TreeBuilder builder;
};
GitRepoImpl & repo;
TreeBuilder rootBuilder;
std::vector<PendingDir> pendingDirs;
/**
@@ -1078,30 +1086,9 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
*/
decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
void pushBuilder(std::string name)
{
const git_tree_entry * entry;
Tree prevTree = nullptr;
if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
/* Clone a tree that we've already finished. This happens
if a tarball has directory entries that are not
contiguous. */
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
throw Error("parent of '%s' is not a directory", name);
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
}
git_treebuilder * b;
if (git_treebuilder_new(&b, *repo, prevTree.get()))
throw Error("creating a tree builder: %s", git_error_last()->message);
pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
};
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
DirectoryState(GitRepoImpl & repo)
: repo(repo)
, rootBuilder(makeBuilder())
{
/* Monkey-patching the pack backend to only read the pack directory
once. Otherwise it will do a readdir for each added oid when it's
@@ -1126,62 +1113,145 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
*/
packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
}
pushBuilder("");
}
std::pair<git_oid, std::string> popBuilder()
{
assert(!pendingDirs.empty());
auto pending = std::move(pendingDirs.back());
git_oid oid;
if (git_treebuilder_write(&oid, pending.builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
pendingDirs.pop_back();
return {oid, pending.name};
};
TreeBuilder makeBuilder(git_tree * prevTree = nullptr);
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
{
assert(!pendingDirs.empty());
auto & pending = pendingDirs.back();
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
};
std::vector<std::string> prepareDirs(const CanonPath & path);
void updateBuilders(std::span<const std::string> names);
std::pair<git_oid, std::string> popBuilder();
void pushBuilder(std::string name);
void addToTree(const std::string & name, const DirEntry & entry);
};
void updateBuilders(std::span<const std::string> names)
{
// Find the common prefix of pendingDirs and names.
size_t prefixLen = 0;
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
break;
TreeBuilder DirectoryState::makeBuilder(git_tree * prevTree)
{
TreeBuilder builder;
if (git_treebuilder_new(Setter(builder), repo, prevTree))
throw Error("creating a tree builder: %s", git_error_last()->message);
return builder;
}
// Finish the builders that are not part of the common prefix.
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
auto [oid, name] = popBuilder();
addToTree(name, oid, GIT_FILEMODE_TREE);
git_oid finishBuilder(TreeBuilder & builder)
{
git_oid oid;
if (git_treebuilder_write(&oid, builder.get()))
throw Error("creating a tree object: %s", git_error_last()->message);
return oid;
}
void DirectoryState::pushBuilder(std::string name)
{
auto & parentBuilder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
const git_tree_entry * entry;
Tree prevTree = nullptr;
if ((entry = git_treebuilder_get(parentBuilder.get(), name.c_str()))) {
/* Clone a tree that we've already finished. This happens
if a tarball has directory entries that are not
contiguous. */
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
throw Error("parent of '%s' is not a directory", name);
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), repo, entry))
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
}
pendingDirs.push_back({
.name = std::move(name),
.builder = makeBuilder(prevTree.get()),
});
}
std::pair<git_oid, std::string> DirectoryState::popBuilder()
{
assert(!pendingDirs.empty());
auto pending = std::move(pendingDirs.back());
pendingDirs.pop_back();
return {finishBuilder(pending.builder), pending.name};
}
void DirectoryState::addToTree(const std::string & name, const DirEntry & entry)
{
auto & builder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &entry.oid, entry.mode))
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
}
void DirectoryState::updateBuilders(std::span<const std::string> names)
{
// Find the common prefix of pendingDirs and names.
size_t prefixLen = 0;
for (; prefixLen < names.size() && prefixLen < pendingDirs.size(); ++prefixLen)
if (names[prefixLen] != pendingDirs[prefixLen].name)
break;
// Finish the builders that are not part of the common prefix.
for (auto n = pendingDirs.size(); n > prefixLen; --n) {
auto [oid, name] = popBuilder();
addToTree(name, DirEntry{.oid = oid, .mode = GIT_FILEMODE_TREE});
}
// Create builders for the new directories.
for (auto n = prefixLen; n < names.size(); ++n)
pushBuilder(names[n]);
}
std::vector<std::string> DirectoryState::prepareDirs(const CanonPath & path)
{
std::vector<std::string> pathComponents;
for (auto & c : path)
pathComponents.emplace_back(c);
updateBuilders(pathComponents);
return pathComponents;
}
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
{
ref<GitRepoImpl> repo;
/**
* Root state:
* - nullopt: nothing created yet
* - DirEntry: root is a single non-directory object (file/symlink)
* - DirectoryState: root is a directory with builders
*/
std::optional<std::variant<DirEntry, DirectoryState>> root;
DirectoryState & ensureDirectoryState()
{
if (!root) {
root.emplace<DirectoryState>(*repo);
}
if (auto * dirState = std::get_if<DirectoryState>(&*root)) {
return *dirState;
}
throw Error("cannot create directory entry: root is not a directory");
}
// Create builders for the new directories.
for (auto n = prefixLen; n < names.size(); ++n)
pushBuilder(names[n]);
};
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
: repo(repo)
{
std::span<const std::string> pathComponents2{pathComponents};
}
updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
return true;
void addEntry(const CanonPath & path, const DirEntry & entry)
{
if (auto res = path.parentAndChild()) {
auto & [parent, child] = *res;
auto & dirState = ensureDirectoryState();
dirState.prepareDirs(parent);
dirState.addToTree(std::string(child), entry);
} else {
if (root)
throw Error("root already exists");
root = entry;
}
}
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
if (!prepareDirs(pathComponents, false))
return;
using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
if (stream)
stream->free(stream);
@@ -1260,60 +1330,68 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
"creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
}
addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
addEntry(
path,
DirEntry{
.oid = oid,
.mode = crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB,
});
}
void createDirectory(const CanonPath & path) override
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
(void) prepareDirs(pathComponents, true);
ensureDirectoryState().prepareDirs(path);
if (callback)
(*callback)(*this, path);
}
void createSymlink(const CanonPath & path, const std::string & target) override
{
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
if (!prepareDirs(pathComponents, false))
return;
git_oid oid;
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
addEntry(
path,
DirEntry{
.oid = oid,
.mode = GIT_FILEMODE_LINK,
});
}
void createHardlink(const CanonPath & path, const CanonPath & target) override
{
std::vector<std::string> pathComponents;
for (auto & c : path)
pathComponents.emplace_back(c);
auto & dirState = ensureDirectoryState();
if (!prepareDirs(pathComponents, false))
return;
auto res = path.parentAndChild();
if (!res)
throw Error("hard link cannot be root file system object");
auto & [parent, name] = *res;
dirState.prepareDirs(parent);
// We can't just look up the path from the start of the root, since
// some parent directories may not have finished yet, so we compute
// a relative path that helps us find the right git_tree_builder or object.
auto relTarget = CanonPath(path).parent()->makeRelative(target);
auto relTarget = parent.makeRelative(target);
auto dir = pendingDirs.rbegin();
auto dir = dirState.pendingDirs.rbegin();
// For each ../ component at the start, go up one directory.
// CanonPath::makeRelative() always puts all .. elements at the start,
// so they're all handled by this loop:
std::string_view relTargetLeft(relTarget);
while (hasPrefix(relTargetLeft, "../")) {
if (dir == pendingDirs.rend())
if (dir == dirState.pendingDirs.rend())
throw Error("invalid hard link target '%s' for path '%s'", target, path);
++dir;
relTargetLeft = relTargetLeft.substr(3);
}
if (dir == pendingDirs.rend())
throw Error("invalid hard link target '%s' for path '%s'", target, path);
// Look up the remainder of the target, starting at the
// top-most `git_treebuilder`.
std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
std::variant<git_treebuilder *, git_oid> curDir{
dir == dirState.pendingDirs.rend() ? dirState.rootBuilder.get() : dir->builder.get()};
Object tree; // needed to keep `entry` alive
const git_tree_entry * entry = nullptr;
@@ -1333,26 +1411,41 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
assert(entry);
addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
dirState.addToTree(
std::string(name),
DirEntry{.oid = *git_tree_entry_id(entry), .mode = git_tree_entry_filemode(entry)});
}
Hash flush() override
git::TreeEntry flush() override
{
updateBuilders({});
if (!root)
throw Error("nothing to flush");
auto [oid, _name] = popBuilder();
auto [oid, mode] = std::visit(
overloaded{
[](DirEntry & entry) -> std::pair<git_oid, git_filemode_t> {
return {entry.oid, entry.mode};
},
[](DirectoryState & dirState) -> std::pair<git_oid, git_filemode_t> {
dirState.updateBuilders({});
return {finishBuilder(dirState.rootBuilder), GIT_FILEMODE_TREE};
},
},
*root);
if (auto * backend = repo->packBackend) {
/* We are done writing blobs, can restore refresh functionality. */
backend->refresh = packfileOdbRefresh;
backend->refresh = dirState.packfileOdbRefresh;
}
repo->flush();
return toHash(oid);
return git::TreeEntry{.mode = static_cast<git::Mode>(mode), .hash = toHash(oid)};
}
};
}
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
{
auto self = ref<GitRepoImpl>(shared_from_this());
+2 -2
View File
@@ -321,12 +321,12 @@ struct GitArchiveInputScheme : InputScheme
auto tarballCache = settings.getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush();
auto treeEntry = parseSink->flush();
act.reset();
TarballInfo tarballInfo{
.treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified};
.treeHash = tarballCache->dereferenceSingletonDirectory(treeEntry.hash), .lastModified = lastModified};
cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});
@@ -2,6 +2,7 @@
#include "nix/fetchers/filtering-source-accessor.hh"
#include "nix/util/fs-sink.hh"
#include "nix/util/git.hh"
namespace nix {
@@ -17,9 +18,9 @@ struct Settings;
struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
{
/**
* Flush builder and return a final Git hash.
* Flush builder and return the final Git tree entry (hash and mode).
*/
virtual Hash flush() = 0;
virtual git::TreeEntry flush() = 0;
};
struct GitAccessorOptions
+2 -2
View File
@@ -182,7 +182,7 @@ static DownloadTarballResult downloadTarball_(
auto tarballCache = settings.getTarballCache();
auto parseSink = tarballCache->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
auto tree = parseSink->flush();
auto treeEntry = parseSink->flush();
act.reset();
@@ -196,7 +196,7 @@ static DownloadTarballResult downloadTarball_(
infoAttrs = cached->value;
} else {
infoAttrs.insert_or_assign("etag", res->etag);
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(tree).gitRev());
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(treeEntry.hash).gitRev());
infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
if (res->immutableUrl)
infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);
-1
View File
@@ -124,7 +124,6 @@ if get_option('benchmarks')
'bench-main.cc',
'derivation-parser-bench.cc',
'ref-scan-bench.cc',
'register-valid-paths-bench.cc',
)
benchmark_exe = executable(
@@ -1,79 +0,0 @@
#include <benchmark/benchmark.h>
#include "nix/store/derivations.hh"
#include "nix/store/local-store.hh"
#include "nix/store/store-open.hh"
#include "nix/util/file-system.hh"
#include "nix/util/hash.hh"
#include "nix/util/tests/test-data.hh"
#ifndef _WIN32
# include <filesystem>
# include <fstream>
using namespace nix;
static void BM_RegisterValidPathsDerivations(benchmark::State & state)
{
const int derivationCount = state.range(0);
for (auto _ : state) {
state.PauseTiming();
auto tmpRoot = createTempDir();
auto realStoreDir = tmpRoot / "nix/store";
std::filesystem::create_directories(realStoreDir);
std::shared_ptr<Store> store = openStore(fmt("local?root=%s", tmpRoot.string()));
auto localStore = std::dynamic_pointer_cast<LocalStore>(store);
if (!localStore)
throw Error("expected local store");
ValidPathInfos infos;
for (int i = 0; i < derivationCount; ++i) {
std::string drvName = fmt("register-valid-paths-bench-%d", i);
auto drvPath = StorePath::random(drvName + ".drv");
Derivation drv;
drv.name = drvName;
drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}});
drv.platform = "x86_64-linux";
drv.builder = "foo";
drv.env["out"] = "";
drv.fillInOutputPaths(*localStore);
auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false);
/* Create an on-disk store object without registering it
in the SQLite DB. LocalFSStore::getFSAccessor(path, false)
allows reading store objects based on their filesystem
presence alone. */
std::ofstream out(realStoreDir / std::string(drvPath.to_string()), std::ios::binary);
out.write(drvContents.data(), drvContents.size());
if (!out)
throw SysError("writing derivation to store");
ValidPathInfo info{drvPath, UnkeyedValidPathInfo(*localStore, Hash::dummy)};
info.narSize = drvContents.size();
infos.emplace(drvPath, std::move(info));
}
state.ResumeTiming();
localStore->registerValidPaths(infos);
state.PauseTiming();
localStore.reset();
store.reset();
std::filesystem::remove_all(tmpRoot);
state.ResumeTiming();
}
state.SetItemsProcessed(state.iterations() * derivationCount);
}
BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10);
#endif
+5 -5
View File
@@ -30,6 +30,8 @@
#include <thread>
#include <regex>
using namespace std::string_literals;
namespace nix {
const unsigned int RETRY_TIME_MS_DEFAULT = 250;
@@ -75,9 +77,8 @@ struct curlFileTransfer : public FileTransfer
CURL * req = 0;
// buffer to accompany the `req` above
char errbuf[CURL_ERROR_SIZE];
bool active = false; // whether the handle has been added to the multi object
bool paused = false; // whether the request has been paused previously
bool enqueued = false; // whether the request has been added the incoming queue
bool active = false; // whether the handle has been added to the multi object
bool paused = false; // whether the request has been paused previously
std::string statusMsg;
unsigned int attempt = 0;
@@ -175,7 +176,7 @@ struct curlFileTransfer : public FileTransfer
curl_easy_cleanup(req);
}
try {
if (!done && enqueued)
if (!done)
fail(FileTransferError(
Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri));
} catch (...) {
@@ -886,7 +887,6 @@ struct curlFileTransfer : public FileTransfer
if (state->isQuitting())
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
state->incoming.push(item);
item->enqueued = true; /* Now any exceptions should be reported via the callback. */
}
wakeupMulti();
@@ -420,7 +420,7 @@ private:
uint64_t queryValidPathId(State & state, const StorePath & path);
uint64_t addValidPath(State & state, const ValidPathInfo & info);
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
void invalidatePath(State & state, const StorePath & path);
+15 -5
View File
@@ -647,7 +647,7 @@ void LocalStore::cacheDrvOutputMapping(
[&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); });
}
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info)
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs)
{
if (info.ca.has_value() && !info.isContentAddressed(*this))
throw Error(
@@ -668,16 +668,17 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info)
efficiently query whether a path is an output of some
derivation. */
if (info.path.isDerivation()) {
auto parsedDrv = readInvalidDerivation(info.path);
auto drv = readInvalidDerivation(info.path);
/* Verify that the output paths in the derivation are correct
(i.e., follow the scheme for computing output paths from
derivations). Note that if this throws an error, then the
DB transaction is rolled back, so the path validity
registration above is undone. */
parsedDrv.checkInvariants(*this, info.path);
if (checkOutputs)
drv.checkInvariants(*this, info.path);
for (auto & i : parsedDrv.outputsAndOptPaths(*this)) {
for (auto & i : drv.outputsAndOptPaths(*this)) {
/* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */
if (i.second.second)
@@ -928,7 +929,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
else
addValidPath(*state, i);
addValidPath(*state, i, false);
paths.insert(i.path);
}
@@ -938,6 +939,15 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
}
/* Check that the derivation outputs are correct. We can't do
this in addValidPath() above, because the references might
not be valid yet. */
for (auto & [_, i] : infos)
if (i.path.isDerivation()) {
// FIXME: inefficient; we already loaded the derivation in addValidPath().
readInvalidDerivation(i.path).checkInvariants(*this, i.path);
}
/* Do a topological sort of the paths. This will throw an
error if a cycle is detected and roll back the
transaction. Cycles can only occur when a derivation
+23 -9
View File
@@ -23,9 +23,9 @@ void Store::computeFSClosure(
bool includeOutputs,
bool includeDerivers)
{
std::function<asio::awaitable<StorePathSet>(const StorePath & path)> queryDeps;
std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
if (flipDirection)
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
StorePathSet res;
StorePathSet referrers;
queryReferrers(path, referrers);
@@ -41,14 +41,12 @@ void Store::computeFSClosure(
for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
if (maybeOutPath && isValidPath(*maybeOutPath))
res.insert(*maybeOutPath);
co_return res;
return res;
};
else
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
StorePathSet res;
auto info = co_await callbackToAwaitable<ref<const ValidPathInfo>>(
[this, path](Callback<ref<const ValidPathInfo>> cb) { queryPathInfo(path, std::move(cb)); });
auto info = fut.get();
for (auto & ref : info->references)
if (ref != path)
res.insert(ref);
@@ -60,9 +58,25 @@ void Store::computeFSClosure(
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
res.insert(*info->deriver);
co_return res;
return res;
};
computeClosure<StorePath>(startPaths, paths_, queryDeps);
computeClosure<StorePath>(
startPaths,
paths_,
[&](const StorePath & path, std::function<void(std::promise<std::set<StorePath>> &)> processEdges) {
std::promise<std::set<StorePath>> promise;
std::function<void(std::future<ref<const ValidPathInfo>>)> getDependencies =
[&](std::future<ref<const ValidPathInfo>> fut) {
try {
promise.set_value(queryDeps(path, fut));
} catch (...) {
promise.set_exception(std::current_exception());
}
};
queryPathInfo(path, getDependencies);
processEdges(promise);
});
}
void Store::computeFSClosure(
+29 -3
View File
@@ -20,8 +20,10 @@ TEST(closure, correctClosure)
set<string> aClosure;
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
computeClosure<string>(
{"A"}, aClosure, [&](const std::string & currentNode) -> asio::awaitable<std::set<std::string>> {
co_return testGraph[currentNode];
{"A"}, aClosure, [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
promise<set<string>> promisedNodes;
promisedNodes.set_value(testGraph[currentNode]);
processEdges(promisedNodes);
});
ASSERT_EQ(aClosure, expectedClosure);
@@ -35,7 +37,31 @@ TEST(closure, properlyHandlesDirectExceptions)
set<string> aClosure;
EXPECT_THROW(
computeClosure<string>(
{"A"}, aClosure, [&](const std::string &) -> asio::awaitable<std::set<std::string>> { throw TestExn(); }),
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) { throw TestExn(); }),
TestExn);
}
TEST(closure, properlyHandlesExceptionsInPromise)
{
struct TestExn
{};
set<string> aClosure;
EXPECT_THROW(
computeClosure<string>(
{"A"},
aClosure,
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
promise<set<string>> promise;
try {
throw TestExn();
} catch (...) {
promise.set_exception(std::current_exception());
}
processEdges(promise);
}),
TestExn);
}
+6 -11
View File
@@ -388,13 +388,14 @@ TEST(openFileEnsureBeneathNoSymlinks, works)
TEST(createAnonymousTempFile, works)
{
auto fd = createAnonymousTempFile();
auto fd_ = fromDescriptorReadOnly(fd.get());
writeFull(fd.get(), "test");
lseek(fd.get(), 0, SEEK_SET);
lseek(fd_, 0, SEEK_SET);
FdSource source{fd.get()};
EXPECT_EQ(source.drain(), "test");
lseek(fd.get(), 0, SEEK_END);
lseek(fd_, 0, SEEK_END);
writeFull(fd.get(), "test");
lseek(fd.get(), 0, SEEK_SET);
lseek(fd_, 0, SEEK_SET);
EXPECT_EQ(source.drain(), "testtest");
}
@@ -405,8 +406,9 @@ TEST(createAnonymousTempFile, works)
TEST(FdSource, restartWorks)
{
auto fd = createAnonymousTempFile();
auto fd_ = fromDescriptorReadOnly(fd.get());
writeFull(fd.get(), "hello world");
lseek(fd.get(), 0, SEEK_SET);
lseek(fd_, 0, SEEK_SET);
FdSource source{fd.get()};
EXPECT_EQ(source.drain(), "hello world");
source.restart();
@@ -414,11 +416,4 @@ TEST(FdSource, restartWorks)
EXPECT_EQ(source.drain(), "");
}
TEST(createTempDir, works)
{
auto tmpDir = createTempDir();
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
ASSERT_TRUE(std::filesystem::is_directory(tmpDir));
}
} // namespace nix
-2
View File
@@ -91,9 +91,7 @@ TEST_F(FSSourceAccessorTest, works)
{
RestoreSink sink(false);
sink.dstPath = tmpDir;
#ifndef _WIN32
sink.dirFd = openDirectory(tmpDir);
#endif
sink.createDirectory(CanonPath("subdir"));
sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); });
sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); });
+11
View File
@@ -54,6 +54,17 @@ std::optional<CanonPath> CanonPath::parent() const
return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
}
std::optional<std::pair<CanonPath, std::string_view>> CanonPath::parentAndChild() const
{
if (isRoot())
return std::nullopt;
auto slash = path.rfind('/');
return {{
CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, slash))),
std::string_view{path}.substr(slash + 1)
}};
}
void CanonPath::pop()
{
assert(!isRoot());
+166 -16
View File
@@ -115,6 +115,9 @@ Path canonPath(PathView path, bool resolveSymlinks)
if (!isAbsolute(path))
throw Error("not an absolute path: '%1%'", path);
// For Windows
auto rootName = std::filesystem::path{path}.root_name();
/* This just exists because we cannot set the target of `remaining`
(the callback parameter) directly to a newly-constructed string,
since it is `std::string_view`. */
@@ -144,6 +147,8 @@ Path canonPath(PathView path, bool resolveSymlinks)
}
});
if (!rootName.empty())
ret = rootName.string() + std::move(ret);
return ret;
}
@@ -375,6 +380,14 @@ void syncParent(const Path & path)
fd.fsync();
}
#ifdef __FreeBSD__
# define MOUNTEDPATHS_PARAM , std::set<Path> & mountedPaths
# define MOUNTEDPATHS_ARG , mountedPaths
#else
# define MOUNTEDPATHS_PARAM
# define MOUNTEDPATHS_ARG
#endif
void recursiveSync(const Path & path)
{
/* If it's a file or symlink, just fsync and return. */
@@ -419,6 +432,129 @@ void recursiveSync(const Path & path)
}
}
static void _deletePath(
Descriptor parentfd,
const std::filesystem::path & path,
uint64_t & bytesFreed,
std::exception_ptr & ex MOUNTEDPATHS_PARAM)
{
#ifndef _WIN32
checkInterrupt();
# ifdef __FreeBSD__
// In case of emergency (unmount fails for some reason) not recurse into mountpoints.
// This prevents us from tearing up the nullfs-mounted nix store.
if (mountedPaths.find(path) != mountedPaths.end()) {
return;
}
# endif
std::string name(path.filename());
assert(name != "." && name != ".." && !name.empty());
struct stat st;
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
if (errno == ENOENT)
return;
throw SysError("getting status of %1%", path);
}
if (!S_ISDIR(st.st_mode)) {
/* We are about to delete a file. Will it likely free space? */
switch (st.st_nlink) {
/* Yes: last link. */
case 1:
bytesFreed += st.st_size;
break;
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
was performed. Instead of checking for real let's assume
it's an optimised file and space will be freed.
In worst case we will double count on freed space for files
with exactly two hardlinks for unoptimised packages.
*/
case 2:
bytesFreed += st.st_size;
break;
/* No: 3+ links. */
default:
break;
}
}
if (S_ISDIR(st.st_mode)) {
/* Make the directory accessible. */
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
throw SysError("chmod %1%", path);
}
int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
if (fd == -1)
throw SysError("opening directory %1%", path);
AutoCloseDir dir(fdopendir(fd));
if (!dir)
throw SysError("opening directory %1%", path);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
checkInterrupt();
std::string childName = dirent->d_name;
if (childName == "." || childName == "..")
continue;
_deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG);
}
if (errno)
throw SysError("reading directory %1%", path);
}
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
if (errno == ENOENT)
return;
try {
throw SysError("cannot unlink %1%", path);
} catch (...) {
if (!ex)
ex = std::current_exception();
else
ignoreExceptionExceptInterrupt();
}
}
#else
// TODO implement
throw UnimplementedError("_deletePath");
#endif
}
static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM)
{
assert(path.is_absolute());
assert(path.parent_path() != path);
AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
if (!dirfd) {
if (errno == ENOENT)
return;
throw SysError("opening directory %s", path.parent_path());
}
std::exception_ptr ex;
_deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG);
if (ex)
std::rethrow_exception(ex);
}
void deletePath(const std::filesystem::path & path)
{
uint64_t dummy;
deletePath(path, dummy);
}
void createDir(const Path & path, mode_t mode)
{
if (mkdir(
@@ -441,6 +577,25 @@ void createDirs(const std::filesystem::path & path)
}
}
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
{
// Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
#ifdef __FreeBSD__
std::set<Path> mountedPaths;
struct statfs * mntbuf;
int count;
if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
throw SysError("getmntinfo");
}
for (int i = 0; i < count; i++) {
mountedPaths.emplace(mntbuf[i].f_mntonname);
}
#endif
bytesFreed = 0;
_deletePath(path, bytesFreed MOUNTEDPATHS_ARG);
}
//////////////////////////////////////////////////////////////////////
AutoDelete::AutoDelete()
@@ -517,6 +672,11 @@ void AutoUnmount::cancel()
//////////////////////////////////////////////////////////////////////
std::filesystem::path defaultTempDir()
{
return getEnvNonEmpty("TMPDIR").value_or("/tmp");
}
std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode)
{
while (1) {
@@ -553,27 +713,17 @@ AutoCloseFD createAnonymousTempFile()
{
AutoCloseFD fd;
#ifdef O_TMPFILE
static std::atomic_flag tmpfileUnsupported{};
if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ {
/* Use O_EXCL, because the file is never supposed to be linked into filesystem. */
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR | O_EXCL, S_IWUSR | S_IRUSR);
if (!fd) {
/* Not supported by the filesystem or the kernel. */
if (errno == EOPNOTSUPP || errno == EISDIR)
tmpfileUnsupported.test_and_set(); /* Set flag and fall through to createTempFile. */
else
throw SysError("creating anonymous temporary file");
} else {
return fd; /* Successfully created. */
}
}
#endif
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR, S_IWUSR | S_IRUSR);
if (!fd)
throw SysError("creating anonymous temporary file");
#else
auto [fd2, path] = createTempFile("nix-anonymous");
if (!fd2)
throw SysError("creating temporary file '%s'", path);
fd = std::move(fd2);
#ifndef _WIN32
# ifndef _WIN32
unlink(requireCString(path)); /* We only care about the file descriptor. */
# endif
#endif
return fd;
}
+15 -26
View File
@@ -70,31 +70,7 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
return dst;
}
#ifndef _WIN32
void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
{
if (path.isRoot()) {
createDirectory(path);
callback(*this, path);
return;
}
createDirectory(path);
assert(dirFd); // If that's not true the above call must have thrown an exception.
RestoreSink dirSink{startFsync};
dirSink.dstPath = append(dstPath, path);
dirSink.dirFd =
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirSink.dirFd)
throw SysError("opening directory '%s'", dirSink.dstPath.string());
callback(dirSink, CanonPath::root);
}
#endif
void RestoreSink::createDirectory(const CanonPath & path)
void RestoreSink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
{
auto p = append(dstPath, path);
@@ -107,6 +83,16 @@ void RestoreSink::createDirectory(const CanonPath & path)
if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
throw SysError("creating directory '%s'", p.string());
if (callback) {
RestoreSink dirSink{startFsync};
dirSink.dstPath =
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (!dirSink.dirFd)
throw SysError("opening directory '%s'", dirSink.dstPath.string());
(*callback)(dirSink, CanonPath::root);
}
return;
}
#endif
@@ -125,7 +111,10 @@ void RestoreSink::createDirectory(const CanonPath & path)
throw SysError("creating directory '%1%'", p.string());
}
#endif
};
if (callback)
(*callback)(*this, path);
}
struct RestoreRegularFile : CreateRegularFileSink
{
-68
View File
@@ -1,68 +0,0 @@
#pragma once
///@file
#include "nix/util/callback.hh"
#include "nix/util/ref.hh"
#include "nix/util/signals.hh"
#include <boost/asio/io_context.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/awaitable.hpp>
#include <boost/asio/use_awaitable.hpp>
#include <boost/asio/associated_cancellation_slot.hpp>
#include <concepts>
namespace nix {
namespace asio = boost::asio;
template<typename T, std::invocable<Callback<T>> F, typename CompletionToken>
auto callbackToAwaitable(F && initiate, CompletionToken && token)
{
return asio::async_initiate<CompletionToken, void(std::future<T>)>(
[initiate = std::forward<F>(initiate)](auto handler) mutable {
auto executor = asio::get_associated_executor(handler);
auto done = std::make_shared<std::atomic<bool>>(false);
auto h = std::make_shared<decltype(handler)>(std::move(handler));
if (auto slot = asio::get_associated_cancellation_slot(*h); slot.is_connected()) {
std::weak_ptr wh = h; /* To handle the cyclic ownership. */
std::weak_ptr wdone = done;
slot.assign([executor, wh, wdone](asio::cancellation_type /*don't care*/) {
auto h = wh.lock();
auto done = wdone.lock();
if (!h || !done || done->exchange(true))
return; /* Gracefully die. */
/* Doesn't need to be kept alive for get_future() since it shares the ownership. */
std::promise<T> p;
p.set_exception(std::make_exception_ptr(Interrupted("interrupted by user")));
asio::post(executor, [h, fut = p.get_future()]() mutable { std::move (*h)(std::move(fut)); });
});
}
initiate(Callback<T>([executor, done, h](std::future<T> fut) mutable {
if (done->exchange(true))
/* Early return for cooperative cancellation. The callback has been caller
later than we've been cancelled. In practice we'll get an error, the handler
has already been posted by the cancellation handler. */
return;
asio::post(executor, [h, fut = std::move(fut)]() mutable { std::move (*h)(std::move(fut)); });
}));
},
std::forward<CompletionToken>(token));
}
/**
* Convert a completion handler callback into a stackless coroutine. The
* callback can be invoked on any thread and the completion handler will be
* marshalled to the coroutines executer.
*/
template<typename T, std::invocable<Callback<T>> F>
asio::awaitable<T> callbackToAwaitable(F && initiate)
{
auto fut = co_await callbackToAwaitable<T>(std::forward<F>(initiate), asio::use_awaitable);
co_return fut.get();
}
} // namespace nix
@@ -238,6 +238,13 @@ public:
return ((std::string_view) path).substr(path.rfind('/') + 1);
}
/**
* Combination of `parent` and `baseName`.
*
* @note that because of the `std::string_view`, we are borrowing for the base name.
*/
std::optional<std::pair<CanonPath, std::string_view>> parentAndChild() const;
bool operator==(const CanonPath & x) const
{
return path == x.path;
+59 -153
View File
@@ -1,167 +1,73 @@
#pragma once
///@file
#include "nix/util/async.hh"
#include "nix/util/ref.hh"
#include <boost/asio/io_context.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/strand.hpp>
#include <boost/asio/detached.hpp>
#include <boost/asio/awaitable.hpp>
#include <boost/asio/co_spawn.hpp>
#include <boost/asio/bind_executor.hpp>
#include <boost/asio/executor_work_guard.hpp>
#include <queue>
#include <set>
#include <future>
#include "nix/util/sync.hh"
using std::set;
namespace nix {
template<typename T>
using GetEdgesAsync = std::function<asio::awaitable<std::set<T>>(const T & elt)>;
template<typename T, typename CompletionToken>
auto computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges, CompletionToken && token)
{
auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) {
auto executor = asio::make_strand(asio::get_associated_executor(handler));
/* Hand-rolled dynamic async graph traversal. ASIO/Cobalt and standard
* C++ will get channels and task groups at some point, but this will
* have to suffice for now. */
struct State : std::enable_shared_from_this<State>
{
decltype(executor) executor;
decltype(getEdges) getEdges;
decltype(handler) handler;
std::set<T> & res;
/**
* Needed to keep the ctx.run() alive because actual work might be happening on another thread
* (like with FileTransfer case).
*/
asio::executor_work_guard<decltype(State::executor)> workGuard;
std::size_t pending = 0;
/**
* Whether the completion handler has been called.
*/
bool done = false;
/**
* Amount of coroutines currently in flight.
*/
std::size_t inFlight = 0;
/**
* Maximum number of concurrent coroutines. Implements primitive rate limiting.
*/
std::size_t maxConcurrent = 64;
/**
* Nodes to handle next.
*/
std::queue<T> todo;
State(
decltype(executor) executor_, decltype(getEdges) getEdges, decltype(handler) handler, std::set<T> & res)
: executor(executor_)
, getEdges(std::move(getEdges))
, handler(std::move(handler))
, res(res)
, workGuard(asio::make_work_guard(executor_))
{
}
void complete(std::exception_ptr ex)
{
if (std::exchange(done, true))
return;
workGuard.reset(); /* We are done and we can release the lock. */
asio::post(executor, [state = this->shared_from_this(), ex] { state->handler(ex); });
}
void enqueue(const std::set<T> & elts)
{
for (const auto & elt : elts)
enqueue(elt);
}
void spawnWorker(const T & elt)
{
++inFlight;
auto state = this->shared_from_this();
asio::post(executor, [state = this->shared_from_this(), elt = std::move(elt)] {
asio::co_spawn(
state->executor,
[state, elt]() -> asio::awaitable<void> {
try {
state->enqueue(co_await state->getEdges(elt));
} catch (...) {
state->complete(std::current_exception());
}
state->onWorkDone();
},
asio::detached);
});
}
void onWorkDone()
{
--inFlight;
--pending;
if (!todo.empty()) {
auto next = std::move(todo.front());
todo.pop();
asio::post(executor, [state = this->shared_from_this(), next = std::move(next)]() mutable {
state->spawnWorker(std::move(next));
});
} else if (pending == 0) {
complete(std::exception_ptr{});
}
}
void enqueue(const T & elt)
{
if (done)
return;
if (!res.insert(elt).second)
return;
++pending;
if (inFlight < maxConcurrent) {
spawnWorker(elt);
} else {
todo.push(elt);
}
}
};
auto state = make_ref<State>(executor, std::move(getEdges), std::move(handler), res);
if (startElts.empty()) {
/* No work to do. */
state->complete(std::exception_ptr{});
return;
}
asio::post(executor, [state, startElts = std::move(startElts)] { state->enqueue(startElts); });
};
return asio::async_initiate<CompletionToken, void(std::exception_ptr)>(std::move(initiator), token);
}
using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
template<typename T>
void computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges)
void computeClosure(const set<T> startElts, set<T> & res, GetEdgesAsync<T> getEdgesAsync)
{
asio::io_context ctx;
std::exception_ptr ex = nullptr;
computeClosure(
std::move(startElts),
res,
std::move(getEdges),
asio::bind_executor(ctx.get_executor(), [&](std::exception_ptr ex2) { ex = ex2; }));
ctx.run();
if (ex)
std::rethrow_exception(ex);
struct State
{
size_t pending;
set<T> & res;
std::exception_ptr exc;
};
Sync<State> state_(State{0, res, 0});
std::condition_variable done;
auto enqueue = [&](this auto & enqueue, const T & current) -> void {
{
auto state(state_.lock());
if (state->exc)
return;
if (!state->res.insert(current).second)
return;
state->pending++;
}
getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
try {
auto children = prom.get_future().get();
for (auto & child : children)
enqueue(child);
{
auto state(state_.lock());
assert(state->pending);
if (!--state->pending)
done.notify_one();
}
} catch (...) {
auto state(state_.lock());
if (!state->exc)
state->exc = std::current_exception();
assert(state->pending);
if (!--state->pending)
done.notify_one();
};
});
};
for (auto & startElt : startElts)
enqueue(startElt);
{
auto state(state_.lock());
while (state->pending)
state.wait(done);
if (state->exc)
std::rethrow_exception(state->exc);
}
}
} // namespace nix
@@ -261,14 +261,4 @@ Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & p
MakeError(EndOfFile, Error);
#ifdef _WIN32
/**
* Windows specific replacement for POSIX `lseek` that operates on a `HANDLE` and not
* a file descriptor.
*/
off_t lseek(Descriptor fd, off_t offset, int whence);
#endif
} // namespace nix
@@ -40,11 +40,6 @@ struct UnixPathTrait
{
return path.rfind('/', from);
}
static size_t rootNameLen(StringView)
{
return 0;
}
};
/**
@@ -88,18 +83,6 @@ struct WindowsPathTrait
size_t p2 = path.rfind(preferredSep, from);
return p1 == String::npos ? p2 : p2 == String::npos ? p1 : std::max(p1, p2);
}
static size_t rootNameLen(StringView path)
{
if (path.size() >= 2 && path[1] == ':') {
char driveLetter = path[0];
if ((driveLetter >= 'A' && driveLetter <= 'Z') || (driveLetter >= 'a' && driveLetter <= 'z'))
return 2;
}
/* TODO: This needs to also handle UNC paths.
* https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats#unc-paths */
return 0;
}
};
template<typename CharT>
@@ -133,11 +116,6 @@ typename PathDict::String canonPathInner(typename PathDict::StringView remaining
typename PathDict::String result;
result.reserve(256);
if (auto rootNameLength = PathDict::rootNameLen(remaining)) {
result += remaining.substr(0, rootNameLength); /* Copy drive letter verbatim. */
remaining.remove_prefix(rootNameLength);
}
while (true) {
/* Skip slashes. */
@@ -362,8 +362,6 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
/**
* Return `TMPDIR`, or the default temporary directory if unset or empty.
* Uses GetTempPathW on windows which respects TMP, TEMP, USERPROFILE env variables.
* Does not resolve symlinks and the returned path might not be directory or exist at all.
*/
std::filesystem::path defaultTempDir();
+14 -14
View File
@@ -34,8 +34,6 @@ struct FileSystemObjectSink
{
virtual ~FileSystemObjectSink() = default;
virtual void createDirectory(const CanonPath & path) = 0;
using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
/**
@@ -47,11 +45,7 @@ struct FileSystemObjectSink
* freshly created directory. Use this when it's important to disallow any
* intermediate path components from being symlinks.
*/
virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
{
createDirectory(path);
callback(*this, path);
}
virtual void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) = 0;
/**
* This function in general is no re-entrant. Only one file can be
@@ -86,7 +80,11 @@ void copyRecursive(
*/
struct NullFileSystemObjectSink : FileSystemObjectSink
{
void createDirectory(const CanonPath & path) override {}
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
{
if (callback)
(*callback)(*this, CanonPath::root);
}
void createSymlink(const CanonPath & path, const std::string & target) override {}
@@ -118,11 +116,7 @@ struct RestoreSink : FileSystemObjectSink
{
}
void createDirectory(const CanonPath & path) override;
#ifndef _WIN32
void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
#endif
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
@@ -144,9 +138,15 @@ struct RegularFileSink : FileSystemObjectSink
{
}
void createDirectory(const CanonPath & path) override
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
{
regular = false;
if (callback) {
NullFileSystemObjectSink s;
/* The children of the
directory that cannot exist also cannot exist */
(*callback)(s, CanonPath::root);
}
}
void createSymlink(const CanonPath & path, const std::string & target) override
@@ -153,7 +153,7 @@ struct MemorySink : FileSystemObjectSink
{
}
void createDirectory(const CanonPath & path) override;
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
-2
View File
@@ -10,7 +10,6 @@ headers = files(
'args.hh',
'args/root.hh',
'array-from-string-literal.hh',
'async.hh',
'base-n.hh',
'base-nix-32.hh',
'callback.hh',
@@ -66,7 +65,6 @@ headers = files(
'signals.hh',
'signature/local-keys.hh',
'signature/signer.hh',
'socket.hh',
'sort.hh',
'source-accessor.hh',
'source-path.hh',
-61
View File
@@ -1,61 +0,0 @@
#pragma once
///@file
#include "nix/util/file-descriptor.hh"
#ifdef _WIN32
# include <winsock2.h>
#endif
namespace nix {
/**
* Often we want to use `Descriptor`, but Windows makes a slightly
* stronger file descriptor vs socket distinction, at least at the level
* of C types.
*/
using Socket =
#ifdef _WIN32
SOCKET
#else
int
#endif
;
#ifdef _WIN32
/**
* Windows gives this a different name
*/
# define SHUT_WR SD_SEND
# define SHUT_RDWR SD_BOTH
#endif
/**
* Convert a `Descriptor` to a `Socket`
*
* This is a no-op except on Windows.
*/
static inline Socket toSocket(Descriptor fd)
{
#ifdef _WIN32
return reinterpret_cast<Socket>(fd);
#else
return fd;
#endif
}
/**
* Convert a `Socket` to a `Descriptor`
*
* This is a no-op except on Windows.
*/
static inline Descriptor fromSocket(Socket fd)
{
#ifdef _WIN32
return reinterpret_cast<Descriptor>(fd);
#else
return fd;
#endif
}
} // namespace nix
@@ -3,8 +3,10 @@
#include "nix/util/types.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/socket.hh"
#ifdef _WIN32
# include <winsock2.h>
#endif
#include <unistd.h>
#include <filesystem>
@@ -21,6 +23,55 @@ AutoCloseFD createUnixDomainSocket();
*/
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
/**
* Often we want to use `Descriptor`, but Windows makes a slightly
* stronger file descriptor vs socket distinction, at least at the level
* of C types.
*/
using Socket =
#ifdef _WIN32
SOCKET
#else
int
#endif
;
#ifdef _WIN32
/**
* Windows gives this a different name
*/
# define SHUT_WR SD_SEND
# define SHUT_RDWR SD_BOTH
#endif
/**
* Convert a `Socket` to a `Descriptor`
*
* This is a no-op except on Windows.
*/
static inline Socket toSocket(Descriptor fd)
{
#ifdef _WIN32
return reinterpret_cast<Socket>(fd);
#else
return fd;
#endif
}
/**
* Convert a `Socket` to a `Descriptor`
*
* This is a no-op except on Windows.
*/
static inline Descriptor fromSocket(Socket fd)
{
#ifdef _WIN32
return reinterpret_cast<Descriptor>(fd);
#else
return fd;
#endif
}
/**
* Bind a Unix domain socket to a path.
*/
+5 -2
View File
@@ -146,7 +146,7 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents
using File = MemorySourceAccessor::File;
void MemorySink::createDirectory(const CanonPath & path)
void MemorySink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
{
auto * f = dst.open(path, File{File::Directory{}});
if (!f)
@@ -154,7 +154,10 @@ void MemorySink::createDirectory(const CanonPath & path)
if (!std::holds_alternative<File::Directory>(f->raw))
throw Error("file '%s' is not a directory", path);
};
if (callback)
(*callback)(*this, path);
}
struct CreateMemoryRegularFile : CreateRegularFileSink
{
+4 -3
View File
@@ -1,5 +1,4 @@
#include "nix/util/nar-accessor.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/archive.hh"
#include <map>
@@ -99,11 +98,13 @@ struct NarAccessor : public SourceAccessor
}
}
void createDirectory(const CanonPath & path) override
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
{
createMember(
path,
NarMember{.stat = {.type = Type::tDirectory, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
if (callback)
(*callback)(*this, path);
}
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
@@ -282,7 +283,7 @@ GetNarBytes seekableGetNarBytes(const Path & path)
GetNarBytes seekableGetNarBytes(Descriptor fd)
{
return [fd](uint64_t offset, uint64_t length) {
if (lseek(fd, offset, SEEK_SET) == -1)
if (::lseek(fromDescriptorReadOnly(fd), offset, SEEK_SET) == -1)
throw SysError("seeking in file");
std::string buf(length, 0);
+8 -8
View File
@@ -1,8 +1,6 @@
#include "nix/util/serialise.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/compression.hh"
#include "nix/util/signals.hh"
#include "nix/util/socket.hh"
#include "nix/util/util.hh"
#include <cstring>
@@ -13,6 +11,7 @@
#ifdef _WIN32
# include <fileapi.h>
# include <winsock2.h>
# include "nix/util/windows-error.hh"
#else
# include <poll.h>
@@ -163,11 +162,11 @@ size_t FdSource::readUnbuffered(char * data, size_t len)
_good = false;
throw SysError("reading from file");
}
#endif
if (n == 0) {
_good = false;
throw EndOfFile(std::string(*endOfFileError));
}
#endif
read += n;
return n;
}
@@ -185,20 +184,20 @@ bool FdSource::hasData()
while (true) {
fd_set fds;
FD_ZERO(&fds);
Socket sock = toSocket(fd);
FD_SET(sock, &fds);
int fd_ = fromDescriptorReadOnly(fd);
FD_SET(fd_, &fds);
struct timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = 0;
auto n = select(sock + 1, &fds, nullptr, nullptr, &timeout);
auto n = select(fd_ + 1, &fds, nullptr, nullptr, &timeout);
if (n < 0) {
if (errno == EINTR)
continue;
throw SysError("polling file descriptor");
}
return FD_ISSET(sock, &fds);
return FD_ISSET(fd, &fds);
}
}
@@ -208,7 +207,8 @@ void FdSource::restart()
throw Error("can't seek to the start of a file");
buffer.reset();
read = bufPosIn = bufPosOut = 0;
if (lseek(fd, 0, SEEK_SET) == -1)
int fd_ = fromDescriptorReadOnly(fd);
if (lseek(fd_, 0, SEEK_SET) == -1)
throw SysError("seeking to the start of a file");
}
-158
View File
@@ -9,9 +9,6 @@
#include <unistd.h>
#include "nix/util/file-system.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/signals.hh"
#include "nix/util/util.hh"
#include "util-unix-config-private.hh"
@@ -22,11 +19,6 @@ Descriptor openDirectory(const std::filesystem::path & path)
return open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
}
std::filesystem::path defaultTempDir()
{
return getEnvNonEmpty("TMPDIR").value_or("/tmp");
}
void setWriteTime(
const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional<bool> optIsSymlink)
{
@@ -74,154 +66,4 @@ void setWriteTime(
#endif
}
#ifdef __FreeBSD__
# define MOUNTEDPATHS_PARAM , std::set<Path> & mountedPaths
# define MOUNTEDPATHS_ARG , mountedPaths
#else
# define MOUNTEDPATHS_PARAM
# define MOUNTEDPATHS_ARG
#endif
static void _deletePath(
Descriptor parentfd,
const std::filesystem::path & path,
uint64_t & bytesFreed,
std::exception_ptr & ex MOUNTEDPATHS_PARAM)
{
#ifndef _WIN32
checkInterrupt();
# ifdef __FreeBSD__
// In case of emergency (unmount fails for some reason) not recurse into mountpoints.
// This prevents us from tearing up the nullfs-mounted nix store.
if (mountedPaths.find(path) != mountedPaths.end()) {
return;
}
# endif
std::string name(path.filename());
assert(name != "." && name != ".." && !name.empty());
struct stat st;
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
if (errno == ENOENT)
return;
throw SysError("getting status of %1%", path);
}
if (!S_ISDIR(st.st_mode)) {
/* We are about to delete a file. Will it likely free space? */
switch (st.st_nlink) {
/* Yes: last link. */
case 1:
bytesFreed += st.st_size;
break;
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
was performed. Instead of checking for real let's assume
it's an optimised file and space will be freed.
In worst case we will double count on freed space for files
with exactly two hardlinks for unoptimised packages.
*/
case 2:
bytesFreed += st.st_size;
break;
/* No: 3+ links. */
default:
break;
}
}
if (S_ISDIR(st.st_mode)) {
/* Make the directory accessible. */
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
throw SysError("chmod %1%", path);
}
int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
if (fd == -1)
throw SysError("opening directory %1%", path);
AutoCloseDir dir(fdopendir(fd));
if (!dir)
throw SysError("opening directory %1%", path);
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
checkInterrupt();
std::string childName = dirent->d_name;
if (childName == "." || childName == "..")
continue;
_deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG);
}
if (errno)
throw SysError("reading directory %1%", path);
}
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
if (errno == ENOENT)
return;
try {
throw SysError("cannot unlink %1%", path);
} catch (...) {
if (!ex)
ex = std::current_exception();
else
ignoreExceptionExceptInterrupt();
}
}
#else
// TODO implement
throw UnimplementedError("_deletePath");
#endif
}
static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM)
{
assert(path.is_absolute());
assert(path.parent_path() != path);
AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
if (!dirfd) {
if (errno == ENOENT)
return;
throw SysError("opening directory %s", path.parent_path());
}
std::exception_ptr ex;
_deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG);
if (ex)
std::rethrow_exception(ex);
}
void deletePath(const std::filesystem::path & path)
{
uint64_t dummy;
deletePath(path, dummy);
}
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
{
// Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
#ifdef __FreeBSD__
std::set<Path> mountedPaths;
struct statfs * mntbuf;
int count;
if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
throw SysError("getmntinfo");
}
for (int i = 0; i < count; i++) {
mountedPaths.emplace(mntbuf[i].f_mntonname);
}
#endif
bytesFreed = 0;
_deletePath(path, bytesFreed MOUNTEDPATHS_ARG);
}
} // namespace nix
+13 -42
View File
@@ -5,12 +5,13 @@
#include "nix/util/windows-error.hh"
#include "nix/util/file-path.hh"
#include <fileapi.h>
#include <error.h>
#include <namedpipeapi.h>
#include <namedpipeapi.h>
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#ifdef _WIN32
# include <fileapi.h>
# include <error.h>
# include <namedpipeapi.h>
# include <namedpipeapi.h>
# define WIN32_LEAN_AND_MEAN
# include <windows.h>
namespace nix {
@@ -45,16 +46,16 @@ void writeFull(HANDLE handle, std::string_view s, bool allowInterrupts)
if (allowInterrupts)
checkInterrupt();
DWORD res;
#if _WIN32_WINNT >= 0x0600
# if _WIN32_WINNT >= 0x0600
auto path = handleToPath(handle); // debug; do it before because handleToPath changes lasterror
if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) {
throw WinError("writing to file %1%:%2%", handle, path);
}
#else
# else
if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) {
throw WinError("writing to file %1%", handle);
}
#endif
# endif
if (res > 0)
s.remove_prefix(res);
}
@@ -119,7 +120,7 @@ void Pipe::create()
//////////////////////////////////////////////////////////////////////
#if _WIN32_WINNT >= 0x0600
# if _WIN32_WINNT >= 0x0600
std::wstring windows::handleToFileName(HANDLE handle)
{
@@ -148,37 +149,7 @@ Path windows::handleToPath(HANDLE handle)
return os_string_to_string(handleToFileName(handle));
}
#endif
off_t lseek(HANDLE h, off_t offset, int whence)
{
DWORD method;
switch (whence) {
case SEEK_SET:
method = FILE_BEGIN;
break;
case SEEK_CUR:
method = FILE_CURRENT;
break;
case SEEK_END:
method = FILE_END;
break;
default:
throw Error("lseek: invalid whence %d", whence);
}
LARGE_INTEGER li;
li.QuadPart = offset;
LARGE_INTEGER newPos;
if (!SetFilePointerEx(h, li, &newPos, method)) {
/* Convert to a POSIX error, since caller code works with this as if it were
a POSIX lseek. */
errno = std::error_code(GetLastError(), std::system_category()).default_error_condition().value();
return -1;
}
return newPos.QuadPart;
}
# endif
} // namespace nix
#endif
+2 -26
View File
@@ -1,11 +1,9 @@
#include "nix/util/file-system.hh"
#include "nix/util/windows-error.hh"
#include "nix/util/logging.hh"
#ifdef _WIN32
namespace nix {
using namespace nix::windows;
void setWriteTime(
const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional<bool> optIsSymlink)
{
@@ -30,27 +28,5 @@ Descriptor openDirectory(const std::filesystem::path & path)
NULL);
}
std::filesystem::path defaultTempDir()
{
wchar_t buf[MAX_PATH + 1];
DWORD len = GetTempPathW(MAX_PATH + 1, buf);
if (len == 0 || len > MAX_PATH)
throw WinError("getting default temporary directory");
return std::filesystem::path(buf);
}
void deletePath(const std::filesystem::path & path)
{
std::error_code ec;
std::filesystem::remove_all(path, ec);
if (ec && ec != std::errc::no_such_file_or_directory)
throw SysError(ec.default_error_condition().value(), "recursively deleting %1%", path);
}
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
{
bytesFreed = 0;
deletePath(path);
}
} // namespace nix
#endif
+5 -6
View File
@@ -437,23 +437,22 @@ static void forwardStdioConnection(RemoteStore & store)
int from = conn->from.fd;
int to = conn->to.fd;
Socket fromSock = toSocket(from), stdinSock = toSocket(getStandardInput());
auto nfds = std::max(fromSock, stdinSock) + 1;
auto nfds = std::max(from, STDIN_FILENO) + 1;
while (true) {
fd_set fds;
FD_ZERO(&fds);
FD_SET(fromSock, &fds);
FD_SET(stdinSock, &fds);
FD_SET(from, &fds);
FD_SET(STDIN_FILENO, &fds);
if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
throw SysError("waiting for data from client or server");
if (FD_ISSET(fromSock, &fds)) {
if (FD_ISSET(from, &fds)) {
auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
if (res == -1)
throw SysError("splicing data from daemon socket to stdout");
else if (res == 0)
throw EndOfFile("unexpected EOF from daemon socket");
}
if (FD_ISSET(stdinSock, &fds)) {
if (FD_ISSET(STDIN_FILENO, &fds)) {
auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
if (res == -1)
throw SysError("splicing data from stdin to daemon socket");