Compare commits
23
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1c0f416cb | ||
|
|
8104858643 | ||
|
|
a6c1d5637a | ||
|
|
27006cc8a9 | ||
|
|
06f21596a0 | ||
|
|
9254fab407 | ||
|
|
994324feda | ||
|
|
1f739961e5 | ||
|
|
cccfa385e6 | ||
|
|
9d2100a165 | ||
|
|
4769f3c0b2 | ||
|
|
ab354dc8f6 | ||
|
|
188cb798ad | ||
|
|
1aa7ab0dcf | ||
|
|
208ed3c538 | ||
|
|
b6add8dcc6 | ||
|
|
79750a3ccc | ||
|
|
30cd9e43e1 | ||
|
|
0695630eb5 | ||
|
|
89dc57f6aa | ||
|
|
f274a7273a | ||
|
|
675656ffba | ||
|
|
a5edc2d921 |
No files matched your search
@@ -26,7 +26,6 @@ bash = find_program('bash', native : true)
|
||||
# HTML manual dependencies (conditional)
|
||||
if get_option('html-manual')
|
||||
mdbook = find_program('mdbook', native : true)
|
||||
rsync = find_program('rsync', required : true, native : true)
|
||||
endif
|
||||
|
||||
pymod = import('python')
|
||||
@@ -126,7 +125,12 @@ if get_option('html-manual')
|
||||
@0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@
|
||||
@0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md
|
||||
sed -e 's|@version@|@3@|g' < @INPUT2@ > @2@/book.toml
|
||||
@4@ -r -L --exclude='*.drv' --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/
|
||||
# Copy source to build directory, excluding the build directory itself
|
||||
# (which is present when built as an individual component).
|
||||
# Use tar with --dereference to copy symlink targets (e.g., JSON examples from tests).
|
||||
(cd @CURRENT_SOURCE_DIR@ && find . -mindepth 1 -maxdepth 1 ! -name build | tar -c --dereference -T - -f -) | (cd @2@ && tar -xf -)
|
||||
chmod -R u+w @2@
|
||||
find @2@ -name '*.drv' -delete
|
||||
(cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3
|
||||
rm -rf @2@/manual
|
||||
mv @2@/html @2@/manual
|
||||
@@ -138,7 +142,6 @@ if get_option('html-manual')
|
||||
mdbook.full_path(),
|
||||
meson.current_build_dir(),
|
||||
meson.project_version(),
|
||||
rsync.full_path(),
|
||||
),
|
||||
],
|
||||
input : [
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
mdbook,
|
||||
jq,
|
||||
python3,
|
||||
rsync,
|
||||
nix-cli,
|
||||
changelog-d,
|
||||
json-schema-for-humans,
|
||||
@@ -54,6 +53,8 @@ mkMesonDerivation (finalAttrs: {
|
||||
../../src/libstore-tests/data/nar-info
|
||||
../../src/libstore-tests/data/build-result
|
||||
../../src/libstore-tests/data/dummy-store
|
||||
# For derivation examples referenced by symlinks in doc/manual/source/protocols/json/schema/
|
||||
../../tests/functional/derivation
|
||||
# Too many different types of files to filter for now
|
||||
../../doc/manual
|
||||
./.
|
||||
@@ -90,7 +91,6 @@ mkMesonDerivation (finalAttrs: {
|
||||
]
|
||||
++ lib.optionals buildHtmlManual [
|
||||
mdbook
|
||||
rsync
|
||||
json-schema-for-humans
|
||||
]
|
||||
++ lib.optionals (!officialRelease && buildHtmlManual) [
|
||||
|
||||
+52
-39
@@ -148,6 +148,15 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
isInternal =
|
||||
dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
|
||||
|
||||
activeComponentNames = lib.listToAttrs (
|
||||
map (c: {
|
||||
name = c.pname or c.name;
|
||||
value = null;
|
||||
}) activeComponents
|
||||
);
|
||||
|
||||
isActiveComponent = name: activeComponentNames ? ${name};
|
||||
|
||||
in
|
||||
{
|
||||
pname = "shell-for-nix";
|
||||
@@ -190,27 +199,19 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
}
|
||||
);
|
||||
|
||||
small =
|
||||
(finalAttrs.finalPackage.withActiveComponents (
|
||||
c:
|
||||
lib.intersectAttrs (lib.genAttrs [
|
||||
"nix-cli"
|
||||
"nix-util-tests"
|
||||
"nix-store-tests"
|
||||
"nix-expr-tests"
|
||||
"nix-fetchers-tests"
|
||||
"nix-flake-tests"
|
||||
"nix-functional-tests"
|
||||
"nix-perl-bindings"
|
||||
] (_: null)) c
|
||||
)).overrideAttrs
|
||||
(o: {
|
||||
mesonFlags = o.mesonFlags ++ [
|
||||
# TODO: infer from activeComponents or vice versa
|
||||
"-Dkaitai-struct-checks=false"
|
||||
"-Djson-schema-checks=false"
|
||||
];
|
||||
});
|
||||
small = finalAttrs.finalPackage.withActiveComponents (
|
||||
c:
|
||||
lib.intersectAttrs (lib.genAttrs [
|
||||
"nix-cli"
|
||||
"nix-util-tests"
|
||||
"nix-store-tests"
|
||||
"nix-expr-tests"
|
||||
"nix-fetchers-tests"
|
||||
"nix-flake-tests"
|
||||
"nix-functional-tests"
|
||||
"nix-perl-bindings"
|
||||
] (_: null)) c
|
||||
);
|
||||
};
|
||||
|
||||
# Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
|
||||
@@ -278,21 +279,33 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
|
||||
dontUseCmakeConfigure = true;
|
||||
|
||||
mesonFlags =
|
||||
map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
|
||||
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
|
||||
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
|
||||
++ lib.optionals havePerl (
|
||||
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
|
||||
)
|
||||
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
|
||||
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
|
||||
mesonFlags = [
|
||||
(lib.mesonBool "kaitai-struct-checks" (isActiveComponent "nix-kaitai-struct-checks"))
|
||||
(lib.mesonBool "json-schema-checks" (isActiveComponent "nix-json-schema-checks"))
|
||||
]
|
||||
++ map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
|
||||
++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)
|
||||
++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents2.nix-fetchers.mesonFlags)
|
||||
++ lib.optionals havePerl (
|
||||
map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
|
||||
)
|
||||
++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
|
||||
++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
|
||||
|
||||
nativeBuildInputs =
|
||||
let
|
||||
inputs =
|
||||
dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
|
||||
lib.filter (x: !isInternal x) (
|
||||
lib.lists.concatMap (
|
||||
# Nix manual has a build-time dependency on nix, but we
|
||||
# don't want to do a native build just to enter the ross
|
||||
# dev shell.
|
||||
#
|
||||
# TODO: think of a more principled fix for this.
|
||||
c: lib.filter (f: f.pname or null != "nix") c.nativeBuildInputs
|
||||
) activeComponents
|
||||
)
|
||||
)
|
||||
++ lib.optional (
|
||||
!buildCanExecuteHost
|
||||
@@ -308,8 +321,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
pkgs.buildPackages.nixfmt-rfc-style
|
||||
pkgs.buildPackages.shellcheck
|
||||
pkgs.buildPackages.include-what-you-use
|
||||
pkgs.buildPackages.gdb
|
||||
]
|
||||
++ lib.optional pkgs.hostPlatform.isUnix pkgs.buildPackages.gdb
|
||||
++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
|
||||
lib.hiPrio pkgs.buildPackages.clang-tools
|
||||
)
|
||||
@@ -325,13 +338,13 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
|
||||
)
|
||||
);
|
||||
|
||||
buildInputs = [
|
||||
pkgs.gbenchmark
|
||||
]
|
||||
++ dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
||||
)
|
||||
++ lib.optional havePerl pkgs.perl;
|
||||
buildInputs =
|
||||
# TODO change Nixpkgs to mark gbenchmark as building on Windows
|
||||
lib.optional pkgs.hostPlatform.isUnix pkgs.gbenchmark
|
||||
++ dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
|
||||
)
|
||||
++ lib.optional havePerl pkgs.perl;
|
||||
|
||||
propagatedBuildInputs = dedupByString (v: "${v}") (
|
||||
lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
|
||||
|
||||
@@ -70,6 +70,12 @@ TEST_F(GitUtilsTest, sink_basic)
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
// TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory
|
||||
// here
|
||||
// The sync method does not document this behavior, should probably renamed because it's not very
|
||||
// general, and I can't imagine that "non-conventional" archives or any other source to be handled by
|
||||
// this sink.
|
||||
|
||||
sink->createDirectory(CanonPath("foo-1.1"));
|
||||
|
||||
sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) {
|
||||
@@ -85,7 +91,7 @@ TEST_F(GitUtilsTest, sink_basic)
|
||||
|
||||
// sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello"));
|
||||
|
||||
auto result = repo->dereferenceSingletonDirectory(sink->flush().hash);
|
||||
auto result = repo->dereferenceSingletonDirectory(sink->flush());
|
||||
auto accessor = repo->getAccessor(result, {}, getRepoName());
|
||||
auto entries = accessor->readDirectory(CanonPath::root);
|
||||
ASSERT_EQ(entries.size(), 5u);
|
||||
@@ -117,51 +123,6 @@ TEST_F(GitUtilsTest, sink_hardlink)
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, sink_root_file)
|
||||
{
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
sink->createRegularFile(
|
||||
CanonPath::root, [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
|
||||
|
||||
auto result = sink->flush();
|
||||
ASSERT_EQ(result.mode, nix::git::Mode::Regular);
|
||||
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
|
||||
ASSERT_EQ(accessor->readFile(CanonPath::root), "hello world");
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, sink_root_symlink)
|
||||
{
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
sink->createSymlink(CanonPath::root, "target");
|
||||
|
||||
auto result = sink->flush();
|
||||
ASSERT_EQ(result.mode, nix::git::Mode::Symlink);
|
||||
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
|
||||
ASSERT_EQ(accessor->readLink(CanonPath::root), "target");
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, sink_hardlink_to_root)
|
||||
{
|
||||
auto repo = openRepo();
|
||||
auto sink = repo->getFileSystemObjectSink();
|
||||
|
||||
sink->createRegularFile(
|
||||
CanonPath("hello"), [](CreateRegularFileSink & fileSink) { writeString(fileSink, "hello world", false); });
|
||||
|
||||
sink->createDirectory(CanonPath("subdir"));
|
||||
sink->createHardlink(CanonPath("subdir/link"), CanonPath("hello"));
|
||||
|
||||
auto result = sink->flush();
|
||||
ASSERT_EQ(result.mode, nix::git::Mode::Directory);
|
||||
auto accessor = repo->getAccessor(result.hash, false, getRepoName());
|
||||
ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world");
|
||||
ASSERT_EQ(accessor->readFile(CanonPath("subdir/link")), "hello world");
|
||||
};
|
||||
|
||||
TEST_F(GitUtilsTest, peel_reference)
|
||||
{
|
||||
// Create a commit in the repo
|
||||
|
||||
+99
-192
@@ -962,15 +962,15 @@ struct GitSourceAccessor : SourceAccessor
|
||||
|
||||
Blob getBlob(State & state, const CanonPath & path, bool expectSymlink)
|
||||
{
|
||||
if (!expectSymlink && git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
|
||||
return dupObject<Blob>((git_blob *) &*state.root);
|
||||
|
||||
auto notExpected = [&]() {
|
||||
throw Error(expectSymlink ? "'%s' is not a symlink" : "'%s' is not a regular file", showPath(path));
|
||||
};
|
||||
|
||||
if (path.isRoot()) {
|
||||
if (git_object_type(state.root.get()) == GIT_OBJECT_BLOB)
|
||||
return dupObject<Blob>((git_blob *) &*state.root);
|
||||
if (path.isRoot())
|
||||
notExpected();
|
||||
}
|
||||
|
||||
auto entry = need(state, path);
|
||||
|
||||
@@ -1054,24 +1054,16 @@ struct GitExportIgnoreSourceAccessor : CachingFilteringSourceAccessor
|
||||
}
|
||||
};
|
||||
|
||||
namespace {
|
||||
|
||||
struct DirEntry
|
||||
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
{
|
||||
git_oid oid;
|
||||
git_filemode_t mode;
|
||||
};
|
||||
ref<GitRepoImpl> repo;
|
||||
|
||||
struct DirectoryState
|
||||
{
|
||||
struct PendingDir
|
||||
{
|
||||
std::string name;
|
||||
TreeBuilder builder;
|
||||
};
|
||||
|
||||
GitRepoImpl & repo;
|
||||
TreeBuilder rootBuilder;
|
||||
std::vector<PendingDir> pendingDirs;
|
||||
|
||||
/**
|
||||
@@ -1086,9 +1078,30 @@ struct DirectoryState
|
||||
*/
|
||||
decltype(::git_odb_backend::refresh) packfileOdbRefresh = nullptr;
|
||||
|
||||
DirectoryState(GitRepoImpl & repo)
|
||||
void pushBuilder(std::string name)
|
||||
{
|
||||
const git_tree_entry * entry;
|
||||
Tree prevTree = nullptr;
|
||||
|
||||
if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) {
|
||||
/* Clone a tree that we've already finished. This happens
|
||||
if a tarball has directory entries that are not
|
||||
contiguous. */
|
||||
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
|
||||
throw Error("parent of '%s' is not a directory", name);
|
||||
|
||||
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry))
|
||||
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
|
||||
}
|
||||
|
||||
git_treebuilder * b;
|
||||
if (git_treebuilder_new(&b, *repo, prevTree.get()))
|
||||
throw Error("creating a tree builder: %s", git_error_last()->message);
|
||||
pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)});
|
||||
};
|
||||
|
||||
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
|
||||
: repo(repo)
|
||||
, rootBuilder(makeBuilder())
|
||||
{
|
||||
/* Monkey-patching the pack backend to only read the pack directory
|
||||
once. Otherwise it will do a readdir for each added oid when it's
|
||||
@@ -1113,145 +1126,62 @@ struct DirectoryState
|
||||
*/
|
||||
packfileOdbRefresh = std::exchange(backend->refresh, nullptr);
|
||||
}
|
||||
pushBuilder("");
|
||||
}
|
||||
|
||||
TreeBuilder makeBuilder(git_tree * prevTree = nullptr);
|
||||
|
||||
std::vector<std::string> prepareDirs(const CanonPath & path);
|
||||
void updateBuilders(std::span<const std::string> names);
|
||||
std::pair<git_oid, std::string> popBuilder();
|
||||
void pushBuilder(std::string name);
|
||||
void addToTree(const std::string & name, const DirEntry & entry);
|
||||
};
|
||||
|
||||
TreeBuilder DirectoryState::makeBuilder(git_tree * prevTree)
|
||||
{
|
||||
TreeBuilder builder;
|
||||
if (git_treebuilder_new(Setter(builder), repo, prevTree))
|
||||
throw Error("creating a tree builder: %s", git_error_last()->message);
|
||||
return builder;
|
||||
}
|
||||
|
||||
git_oid finishBuilder(TreeBuilder & builder)
|
||||
{
|
||||
git_oid oid;
|
||||
if (git_treebuilder_write(&oid, builder.get()))
|
||||
throw Error("creating a tree object: %s", git_error_last()->message);
|
||||
return oid;
|
||||
}
|
||||
|
||||
void DirectoryState::pushBuilder(std::string name)
|
||||
{
|
||||
auto & parentBuilder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
|
||||
const git_tree_entry * entry;
|
||||
Tree prevTree = nullptr;
|
||||
|
||||
if ((entry = git_treebuilder_get(parentBuilder.get(), name.c_str()))) {
|
||||
/* Clone a tree that we've already finished. This happens
|
||||
if a tarball has directory entries that are not
|
||||
contiguous. */
|
||||
if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
|
||||
throw Error("parent of '%s' is not a directory", name);
|
||||
|
||||
if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), repo, entry))
|
||||
throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
|
||||
}
|
||||
|
||||
pendingDirs.push_back({
|
||||
.name = std::move(name),
|
||||
.builder = makeBuilder(prevTree.get()),
|
||||
});
|
||||
}
|
||||
|
||||
std::pair<git_oid, std::string> DirectoryState::popBuilder()
|
||||
{
|
||||
assert(!pendingDirs.empty());
|
||||
auto pending = std::move(pendingDirs.back());
|
||||
pendingDirs.pop_back();
|
||||
return {finishBuilder(pending.builder), pending.name};
|
||||
}
|
||||
|
||||
void DirectoryState::addToTree(const std::string & name, const DirEntry & entry)
|
||||
{
|
||||
auto & builder = pendingDirs.empty() ? rootBuilder : pendingDirs.back().builder;
|
||||
if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &entry.oid, entry.mode))
|
||||
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
|
||||
}
|
||||
|
||||
void DirectoryState::updateBuilders(std::span<const std::string> names)
|
||||
{
|
||||
// Find the common prefix of pendingDirs and names.
|
||||
size_t prefixLen = 0;
|
||||
for (; prefixLen < names.size() && prefixLen < pendingDirs.size(); ++prefixLen)
|
||||
if (names[prefixLen] != pendingDirs[prefixLen].name)
|
||||
break;
|
||||
|
||||
// Finish the builders that are not part of the common prefix.
|
||||
for (auto n = pendingDirs.size(); n > prefixLen; --n) {
|
||||
auto [oid, name] = popBuilder();
|
||||
addToTree(name, DirEntry{.oid = oid, .mode = GIT_FILEMODE_TREE});
|
||||
}
|
||||
|
||||
// Create builders for the new directories.
|
||||
for (auto n = prefixLen; n < names.size(); ++n)
|
||||
pushBuilder(names[n]);
|
||||
}
|
||||
|
||||
std::vector<std::string> DirectoryState::prepareDirs(const CanonPath & path)
|
||||
{
|
||||
std::vector<std::string> pathComponents;
|
||||
for (auto & c : path)
|
||||
pathComponents.emplace_back(c);
|
||||
|
||||
updateBuilders(pathComponents);
|
||||
|
||||
return pathComponents;
|
||||
}
|
||||
|
||||
struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
{
|
||||
ref<GitRepoImpl> repo;
|
||||
|
||||
/**
|
||||
* Root state:
|
||||
* - nullopt: nothing created yet
|
||||
* - DirEntry: root is a single non-directory object (file/symlink)
|
||||
* - DirectoryState: root is a directory with builders
|
||||
*/
|
||||
std::optional<std::variant<DirEntry, DirectoryState>> root;
|
||||
|
||||
DirectoryState & ensureDirectoryState()
|
||||
std::pair<git_oid, std::string> popBuilder()
|
||||
{
|
||||
if (!root) {
|
||||
root.emplace<DirectoryState>(*repo);
|
||||
}
|
||||
if (auto * dirState = std::get_if<DirectoryState>(&*root)) {
|
||||
return *dirState;
|
||||
}
|
||||
throw Error("cannot create directory entry: root is not a directory");
|
||||
}
|
||||
assert(!pendingDirs.empty());
|
||||
auto pending = std::move(pendingDirs.back());
|
||||
git_oid oid;
|
||||
if (git_treebuilder_write(&oid, pending.builder.get()))
|
||||
throw Error("creating a tree object: %s", git_error_last()->message);
|
||||
pendingDirs.pop_back();
|
||||
return {oid, pending.name};
|
||||
};
|
||||
|
||||
GitFileSystemObjectSinkImpl(ref<GitRepoImpl> repo)
|
||||
: repo(repo)
|
||||
void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode)
|
||||
{
|
||||
}
|
||||
assert(!pendingDirs.empty());
|
||||
auto & pending = pendingDirs.back();
|
||||
if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode))
|
||||
throw Error("adding a file to a tree builder: %s", git_error_last()->message);
|
||||
};
|
||||
|
||||
void addEntry(const CanonPath & path, const DirEntry & entry)
|
||||
void updateBuilders(std::span<const std::string> names)
|
||||
{
|
||||
if (auto res = path.parentAndChild()) {
|
||||
auto & [parent, child] = *res;
|
||||
auto & dirState = ensureDirectoryState();
|
||||
dirState.prepareDirs(parent);
|
||||
dirState.addToTree(std::string(child), entry);
|
||||
} else {
|
||||
if (root)
|
||||
throw Error("root already exists");
|
||||
root = entry;
|
||||
// Find the common prefix of pendingDirs and names.
|
||||
size_t prefixLen = 0;
|
||||
for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen)
|
||||
if (names[prefixLen] != pendingDirs[prefixLen + 1].name)
|
||||
break;
|
||||
|
||||
// Finish the builders that are not part of the common prefix.
|
||||
for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) {
|
||||
auto [oid, name] = popBuilder();
|
||||
addToTree(name, oid, GIT_FILEMODE_TREE);
|
||||
}
|
||||
|
||||
// Create builders for the new directories.
|
||||
for (auto n = prefixLen; n < names.size(); ++n)
|
||||
pushBuilder(names[n]);
|
||||
};
|
||||
|
||||
bool prepareDirs(const std::vector<std::string> & pathComponents, bool isDir)
|
||||
{
|
||||
std::span<const std::string> pathComponents2{pathComponents};
|
||||
|
||||
updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
using WriteStream = std::unique_ptr<::git_writestream, decltype([](::git_writestream * stream) {
|
||||
if (stream)
|
||||
stream->free(stream);
|
||||
@@ -1330,68 +1260,60 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
"creating a blob object for '%s' from in-memory buffer: %s", path, git_error_last()->message);
|
||||
}
|
||||
|
||||
addEntry(
|
||||
path,
|
||||
DirEntry{
|
||||
.oid = oid,
|
||||
.mode = crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB,
|
||||
});
|
||||
addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB);
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
ensureDirectoryState().prepareDirs(path);
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
(void) prepareDirs(pathComponents, true);
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
{
|
||||
auto pathComponents = tokenizeString<std::vector<std::string>>(path.rel(), "/");
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
git_oid oid;
|
||||
if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size()))
|
||||
throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message);
|
||||
|
||||
addEntry(
|
||||
path,
|
||||
DirEntry{
|
||||
.oid = oid,
|
||||
.mode = GIT_FILEMODE_LINK,
|
||||
});
|
||||
addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK);
|
||||
}
|
||||
|
||||
void createHardlink(const CanonPath & path, const CanonPath & target) override
|
||||
{
|
||||
auto & dirState = ensureDirectoryState();
|
||||
std::vector<std::string> pathComponents;
|
||||
for (auto & c : path)
|
||||
pathComponents.emplace_back(c);
|
||||
|
||||
auto res = path.parentAndChild();
|
||||
if (!res)
|
||||
throw Error("hard link cannot be root file system object");
|
||||
auto & [parent, name] = *res;
|
||||
|
||||
dirState.prepareDirs(parent);
|
||||
if (!prepareDirs(pathComponents, false))
|
||||
return;
|
||||
|
||||
// We can't just look up the path from the start of the root, since
|
||||
// some parent directories may not have finished yet, so we compute
|
||||
// a relative path that helps us find the right git_tree_builder or object.
|
||||
auto relTarget = parent.makeRelative(target);
|
||||
auto relTarget = CanonPath(path).parent()->makeRelative(target);
|
||||
|
||||
auto dir = dirState.pendingDirs.rbegin();
|
||||
auto dir = pendingDirs.rbegin();
|
||||
|
||||
// For each ../ component at the start, go up one directory.
|
||||
// CanonPath::makeRelative() always puts all .. elements at the start,
|
||||
// so they're all handled by this loop:
|
||||
std::string_view relTargetLeft(relTarget);
|
||||
while (hasPrefix(relTargetLeft, "../")) {
|
||||
if (dir == dirState.pendingDirs.rend())
|
||||
if (dir == pendingDirs.rend())
|
||||
throw Error("invalid hard link target '%s' for path '%s'", target, path);
|
||||
++dir;
|
||||
relTargetLeft = relTargetLeft.substr(3);
|
||||
}
|
||||
if (dir == pendingDirs.rend())
|
||||
throw Error("invalid hard link target '%s' for path '%s'", target, path);
|
||||
|
||||
// Look up the remainder of the target, starting at the
|
||||
// top-most `git_treebuilder`.
|
||||
std::variant<git_treebuilder *, git_oid> curDir{
|
||||
dir == dirState.pendingDirs.rend() ? dirState.rootBuilder.get() : dir->builder.get()};
|
||||
std::variant<git_treebuilder *, git_oid> curDir{dir->builder.get()};
|
||||
Object tree; // needed to keep `entry` alive
|
||||
const git_tree_entry * entry = nullptr;
|
||||
|
||||
@@ -1411,41 +1333,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
|
||||
|
||||
assert(entry);
|
||||
|
||||
dirState.addToTree(
|
||||
std::string(name),
|
||||
DirEntry{.oid = *git_tree_entry_id(entry), .mode = git_tree_entry_filemode(entry)});
|
||||
addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry));
|
||||
}
|
||||
|
||||
git::TreeEntry flush() override
|
||||
Hash flush() override
|
||||
{
|
||||
if (!root)
|
||||
throw Error("nothing to flush");
|
||||
updateBuilders({});
|
||||
|
||||
auto [oid, mode] = std::visit(
|
||||
overloaded{
|
||||
[](DirEntry & entry) -> std::pair<git_oid, git_filemode_t> {
|
||||
return {entry.oid, entry.mode};
|
||||
},
|
||||
[](DirectoryState & dirState) -> std::pair<git_oid, git_filemode_t> {
|
||||
dirState.updateBuilders({});
|
||||
return {finishBuilder(dirState.rootBuilder), GIT_FILEMODE_TREE};
|
||||
},
|
||||
},
|
||||
*root);
|
||||
auto [oid, _name] = popBuilder();
|
||||
|
||||
if (auto * backend = repo->packBackend) {
|
||||
/* We are done writing blobs, can restore refresh functionality. */
|
||||
backend->refresh = dirState.packfileOdbRefresh;
|
||||
backend->refresh = packfileOdbRefresh;
|
||||
}
|
||||
|
||||
repo->flush();
|
||||
|
||||
return git::TreeEntry{.mode = static_cast<git::Mode>(mode), .hash = toHash(oid)};
|
||||
return toHash(oid);
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev, const GitAccessorOptions & options)
|
||||
{
|
||||
auto self = ref<GitRepoImpl>(shared_from_this());
|
||||
|
||||
@@ -321,12 +321,12 @@ struct GitArchiveInputScheme : InputScheme
|
||||
auto tarballCache = settings.getTarballCache();
|
||||
auto parseSink = tarballCache->getFileSystemObjectSink();
|
||||
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
||||
auto treeEntry = parseSink->flush();
|
||||
auto tree = parseSink->flush();
|
||||
|
||||
act.reset();
|
||||
|
||||
TarballInfo tarballInfo{
|
||||
.treeHash = tarballCache->dereferenceSingletonDirectory(treeEntry.hash), .lastModified = lastModified};
|
||||
.treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified};
|
||||
|
||||
cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}});
|
||||
cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}});
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
#include "nix/fetchers/filtering-source-accessor.hh"
|
||||
#include "nix/util/fs-sink.hh"
|
||||
#include "nix/util/git.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -18,9 +17,9 @@ struct Settings;
|
||||
struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink
|
||||
{
|
||||
/**
|
||||
* Flush builder and return the final Git tree entry (hash and mode).
|
||||
* Flush builder and return a final Git hash.
|
||||
*/
|
||||
virtual git::TreeEntry flush() = 0;
|
||||
virtual Hash flush() = 0;
|
||||
};
|
||||
|
||||
struct GitAccessorOptions
|
||||
|
||||
@@ -182,7 +182,7 @@ static DownloadTarballResult downloadTarball_(
|
||||
auto tarballCache = settings.getTarballCache();
|
||||
auto parseSink = tarballCache->getFileSystemObjectSink();
|
||||
auto lastModified = unpackTarfileToSink(archive, *parseSink);
|
||||
auto treeEntry = parseSink->flush();
|
||||
auto tree = parseSink->flush();
|
||||
|
||||
act.reset();
|
||||
|
||||
@@ -196,7 +196,7 @@ static DownloadTarballResult downloadTarball_(
|
||||
infoAttrs = cached->value;
|
||||
} else {
|
||||
infoAttrs.insert_or_assign("etag", res->etag);
|
||||
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(treeEntry.hash).gitRev());
|
||||
infoAttrs.insert_or_assign("treeHash", tarballCache->dereferenceSingletonDirectory(tree).gitRev());
|
||||
infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified));
|
||||
if (res->immutableUrl)
|
||||
infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl);
|
||||
|
||||
@@ -124,6 +124,7 @@ if get_option('benchmarks')
|
||||
'bench-main.cc',
|
||||
'derivation-parser-bench.cc',
|
||||
'ref-scan-bench.cc',
|
||||
'register-valid-paths-bench.cc',
|
||||
)
|
||||
|
||||
benchmark_exe = executable(
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
#include <benchmark/benchmark.h>
|
||||
|
||||
#include "nix/store/derivations.hh"
|
||||
#include "nix/store/local-store.hh"
|
||||
#include "nix/store/store-open.hh"
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/hash.hh"
|
||||
#include "nix/util/tests/test-data.hh"
|
||||
|
||||
#ifndef _WIN32
|
||||
|
||||
# include <filesystem>
|
||||
# include <fstream>
|
||||
|
||||
using namespace nix;
|
||||
|
||||
static void BM_RegisterValidPathsDerivations(benchmark::State & state)
|
||||
{
|
||||
const int derivationCount = state.range(0);
|
||||
|
||||
for (auto _ : state) {
|
||||
state.PauseTiming();
|
||||
|
||||
auto tmpRoot = createTempDir();
|
||||
auto realStoreDir = tmpRoot / "nix/store";
|
||||
std::filesystem::create_directories(realStoreDir);
|
||||
|
||||
std::shared_ptr<Store> store = openStore(fmt("local?root=%s", tmpRoot.string()));
|
||||
auto localStore = std::dynamic_pointer_cast<LocalStore>(store);
|
||||
if (!localStore)
|
||||
throw Error("expected local store");
|
||||
|
||||
ValidPathInfos infos;
|
||||
for (int i = 0; i < derivationCount; ++i) {
|
||||
std::string drvName = fmt("register-valid-paths-bench-%d", i);
|
||||
auto drvPath = StorePath::random(drvName + ".drv");
|
||||
|
||||
Derivation drv;
|
||||
drv.name = drvName;
|
||||
drv.outputs.emplace("out", DerivationOutput{DerivationOutput::Deferred{}});
|
||||
drv.platform = "x86_64-linux";
|
||||
drv.builder = "foo";
|
||||
drv.env["out"] = "";
|
||||
drv.fillInOutputPaths(*localStore);
|
||||
|
||||
auto drvContents = drv.unparse(*localStore, /*maskOutputs=*/false);
|
||||
|
||||
/* Create an on-disk store object without registering it
|
||||
in the SQLite DB. LocalFSStore::getFSAccessor(path, false)
|
||||
allows reading store objects based on their filesystem
|
||||
presence alone. */
|
||||
std::ofstream out(realStoreDir / std::string(drvPath.to_string()), std::ios::binary);
|
||||
out.write(drvContents.data(), drvContents.size());
|
||||
if (!out)
|
||||
throw SysError("writing derivation to store");
|
||||
|
||||
ValidPathInfo info{drvPath, UnkeyedValidPathInfo(*localStore, Hash::dummy)};
|
||||
info.narSize = drvContents.size();
|
||||
|
||||
infos.emplace(drvPath, std::move(info));
|
||||
}
|
||||
|
||||
state.ResumeTiming();
|
||||
|
||||
localStore->registerValidPaths(infos);
|
||||
|
||||
state.PauseTiming();
|
||||
localStore.reset();
|
||||
store.reset();
|
||||
std::filesystem::remove_all(tmpRoot);
|
||||
state.ResumeTiming();
|
||||
}
|
||||
|
||||
state.SetItemsProcessed(state.iterations() * derivationCount);
|
||||
}
|
||||
|
||||
BENCHMARK(BM_RegisterValidPathsDerivations)->Arg(10);
|
||||
|
||||
#endif
|
||||
@@ -30,8 +30,6 @@
|
||||
#include <thread>
|
||||
#include <regex>
|
||||
|
||||
using namespace std::string_literals;
|
||||
|
||||
namespace nix {
|
||||
|
||||
const unsigned int RETRY_TIME_MS_DEFAULT = 250;
|
||||
@@ -77,8 +75,9 @@ struct curlFileTransfer : public FileTransfer
|
||||
CURL * req = 0;
|
||||
// buffer to accompany the `req` above
|
||||
char errbuf[CURL_ERROR_SIZE];
|
||||
bool active = false; // whether the handle has been added to the multi object
|
||||
bool paused = false; // whether the request has been paused previously
|
||||
bool active = false; // whether the handle has been added to the multi object
|
||||
bool paused = false; // whether the request has been paused previously
|
||||
bool enqueued = false; // whether the request has been added the incoming queue
|
||||
std::string statusMsg;
|
||||
|
||||
unsigned int attempt = 0;
|
||||
@@ -176,7 +175,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
curl_easy_cleanup(req);
|
||||
}
|
||||
try {
|
||||
if (!done)
|
||||
if (!done && enqueued)
|
||||
fail(FileTransferError(
|
||||
Interrupted, {}, "%s of '%s' was interrupted", Uncolored(request.noun()), request.uri));
|
||||
} catch (...) {
|
||||
@@ -887,6 +886,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (state->isQuitting())
|
||||
throw nix::Error("cannot enqueue download request because the download thread is shutting down");
|
||||
state->incoming.push(item);
|
||||
item->enqueued = true; /* Now any exceptions should be reported via the callback. */
|
||||
}
|
||||
|
||||
wakeupMulti();
|
||||
|
||||
@@ -420,7 +420,7 @@ private:
|
||||
|
||||
uint64_t queryValidPathId(State & state, const StorePath & path);
|
||||
|
||||
uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true);
|
||||
uint64_t addValidPath(State & state, const ValidPathInfo & info);
|
||||
|
||||
void invalidatePath(State & state, const StorePath & path);
|
||||
|
||||
|
||||
@@ -647,7 +647,7 @@ void LocalStore::cacheDrvOutputMapping(
|
||||
[&]() { state.stmts->AddDerivationOutput.use()(deriver)(outputName) (printStorePath(output)).exec(); });
|
||||
}
|
||||
|
||||
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs)
|
||||
uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info)
|
||||
{
|
||||
if (info.ca.has_value() && !info.isContentAddressed(*this))
|
||||
throw Error(
|
||||
@@ -668,17 +668,16 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo
|
||||
efficiently query whether a path is an output of some
|
||||
derivation. */
|
||||
if (info.path.isDerivation()) {
|
||||
auto drv = readInvalidDerivation(info.path);
|
||||
auto parsedDrv = readInvalidDerivation(info.path);
|
||||
|
||||
/* Verify that the output paths in the derivation are correct
|
||||
(i.e., follow the scheme for computing output paths from
|
||||
derivations). Note that if this throws an error, then the
|
||||
DB transaction is rolled back, so the path validity
|
||||
registration above is undone. */
|
||||
if (checkOutputs)
|
||||
drv.checkInvariants(*this, info.path);
|
||||
parsedDrv.checkInvariants(*this, info.path);
|
||||
|
||||
for (auto & i : drv.outputsAndOptPaths(*this)) {
|
||||
for (auto & i : parsedDrv.outputsAndOptPaths(*this)) {
|
||||
/* Floating CA derivations have indeterminate output paths until
|
||||
they are built, so don't register anything in that case */
|
||||
if (i.second.second)
|
||||
@@ -929,7 +928,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||
if (isValidPath_(*state, i.path))
|
||||
updatePathInfo(*state, i);
|
||||
else
|
||||
addValidPath(*state, i, false);
|
||||
addValidPath(*state, i);
|
||||
paths.insert(i.path);
|
||||
}
|
||||
|
||||
@@ -939,15 +938,6 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||
state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
|
||||
}
|
||||
|
||||
/* Check that the derivation outputs are correct. We can't do
|
||||
this in addValidPath() above, because the references might
|
||||
not be valid yet. */
|
||||
for (auto & [_, i] : infos)
|
||||
if (i.path.isDerivation()) {
|
||||
// FIXME: inefficient; we already loaded the derivation in addValidPath().
|
||||
readInvalidDerivation(i.path).checkInvariants(*this, i.path);
|
||||
}
|
||||
|
||||
/* Do a topological sort of the paths. This will throw an
|
||||
error if a cycle is detected and roll back the
|
||||
transaction. Cycles can only occur when a derivation
|
||||
|
||||
+9
-23
@@ -23,9 +23,9 @@ void Store::computeFSClosure(
|
||||
bool includeOutputs,
|
||||
bool includeDerivers)
|
||||
{
|
||||
std::function<std::set<StorePath>(const StorePath & path, std::future<ref<const ValidPathInfo>> &)> queryDeps;
|
||||
std::function<asio::awaitable<StorePathSet>(const StorePath & path)> queryDeps;
|
||||
if (flipDirection)
|
||||
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
|
||||
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
|
||||
StorePathSet res;
|
||||
StorePathSet referrers;
|
||||
queryReferrers(path, referrers);
|
||||
@@ -41,12 +41,14 @@ void Store::computeFSClosure(
|
||||
for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path))
|
||||
if (maybeOutPath && isValidPath(*maybeOutPath))
|
||||
res.insert(*maybeOutPath);
|
||||
return res;
|
||||
co_return res;
|
||||
};
|
||||
else
|
||||
queryDeps = [&](const StorePath & path, std::future<ref<const ValidPathInfo>> & fut) {
|
||||
queryDeps = [this, includeOutputs, includeDerivers](const StorePath & path) -> asio::awaitable<StorePathSet> {
|
||||
StorePathSet res;
|
||||
auto info = fut.get();
|
||||
auto info = co_await callbackToAwaitable<ref<const ValidPathInfo>>(
|
||||
[this, path](Callback<ref<const ValidPathInfo>> cb) { queryPathInfo(path, std::move(cb)); });
|
||||
|
||||
for (auto & ref : info->references)
|
||||
if (ref != path)
|
||||
res.insert(ref);
|
||||
@@ -58,25 +60,9 @@ void Store::computeFSClosure(
|
||||
|
||||
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
|
||||
res.insert(*info->deriver);
|
||||
return res;
|
||||
co_return res;
|
||||
};
|
||||
|
||||
computeClosure<StorePath>(
|
||||
startPaths,
|
||||
paths_,
|
||||
[&](const StorePath & path, std::function<void(std::promise<std::set<StorePath>> &)> processEdges) {
|
||||
std::promise<std::set<StorePath>> promise;
|
||||
std::function<void(std::future<ref<const ValidPathInfo>>)> getDependencies =
|
||||
[&](std::future<ref<const ValidPathInfo>> fut) {
|
||||
try {
|
||||
promise.set_value(queryDeps(path, fut));
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
};
|
||||
queryPathInfo(path, getDependencies);
|
||||
processEdges(promise);
|
||||
});
|
||||
computeClosure<StorePath>(startPaths, paths_, queryDeps);
|
||||
}
|
||||
|
||||
void Store::computeFSClosure(
|
||||
|
||||
@@ -20,10 +20,8 @@ TEST(closure, correctClosure)
|
||||
set<string> aClosure;
|
||||
set<string> expectedClosure = {"A", "B", "C", "F", "G"};
|
||||
computeClosure<string>(
|
||||
{"A"}, aClosure, [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
|
||||
promise<set<string>> promisedNodes;
|
||||
promisedNodes.set_value(testGraph[currentNode]);
|
||||
processEdges(promisedNodes);
|
||||
{"A"}, aClosure, [&](const std::string & currentNode) -> asio::awaitable<std::set<std::string>> {
|
||||
co_return testGraph[currentNode];
|
||||
});
|
||||
|
||||
ASSERT_EQ(aClosure, expectedClosure);
|
||||
@@ -37,31 +35,7 @@ TEST(closure, properlyHandlesDirectExceptions)
|
||||
set<string> aClosure;
|
||||
EXPECT_THROW(
|
||||
computeClosure<string>(
|
||||
{"A"},
|
||||
aClosure,
|
||||
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) { throw TestExn(); }),
|
||||
TestExn);
|
||||
}
|
||||
|
||||
TEST(closure, properlyHandlesExceptionsInPromise)
|
||||
{
|
||||
struct TestExn
|
||||
{};
|
||||
|
||||
set<string> aClosure;
|
||||
EXPECT_THROW(
|
||||
computeClosure<string>(
|
||||
{"A"},
|
||||
aClosure,
|
||||
[&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
|
||||
promise<set<string>> promise;
|
||||
try {
|
||||
throw TestExn();
|
||||
} catch (...) {
|
||||
promise.set_exception(std::current_exception());
|
||||
}
|
||||
processEdges(promise);
|
||||
}),
|
||||
{"A"}, aClosure, [&](const std::string &) -> asio::awaitable<std::set<std::string>> { throw TestExn(); }),
|
||||
TestExn);
|
||||
}
|
||||
|
||||
|
||||
@@ -388,14 +388,13 @@ TEST(openFileEnsureBeneathNoSymlinks, works)
|
||||
TEST(createAnonymousTempFile, works)
|
||||
{
|
||||
auto fd = createAnonymousTempFile();
|
||||
auto fd_ = fromDescriptorReadOnly(fd.get());
|
||||
writeFull(fd.get(), "test");
|
||||
lseek(fd_, 0, SEEK_SET);
|
||||
lseek(fd.get(), 0, SEEK_SET);
|
||||
FdSource source{fd.get()};
|
||||
EXPECT_EQ(source.drain(), "test");
|
||||
lseek(fd_, 0, SEEK_END);
|
||||
lseek(fd.get(), 0, SEEK_END);
|
||||
writeFull(fd.get(), "test");
|
||||
lseek(fd_, 0, SEEK_SET);
|
||||
lseek(fd.get(), 0, SEEK_SET);
|
||||
EXPECT_EQ(source.drain(), "testtest");
|
||||
}
|
||||
|
||||
@@ -406,9 +405,8 @@ TEST(createAnonymousTempFile, works)
|
||||
TEST(FdSource, restartWorks)
|
||||
{
|
||||
auto fd = createAnonymousTempFile();
|
||||
auto fd_ = fromDescriptorReadOnly(fd.get());
|
||||
writeFull(fd.get(), "hello world");
|
||||
lseek(fd_, 0, SEEK_SET);
|
||||
lseek(fd.get(), 0, SEEK_SET);
|
||||
FdSource source{fd.get()};
|
||||
EXPECT_EQ(source.drain(), "hello world");
|
||||
source.restart();
|
||||
@@ -416,4 +414,11 @@ TEST(FdSource, restartWorks)
|
||||
EXPECT_EQ(source.drain(), "");
|
||||
}
|
||||
|
||||
TEST(createTempDir, works)
|
||||
{
|
||||
auto tmpDir = createTempDir();
|
||||
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
|
||||
ASSERT_TRUE(std::filesystem::is_directory(tmpDir));
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -91,7 +91,9 @@ TEST_F(FSSourceAccessorTest, works)
|
||||
{
|
||||
RestoreSink sink(false);
|
||||
sink.dstPath = tmpDir;
|
||||
#ifndef _WIN32
|
||||
sink.dirFd = openDirectory(tmpDir);
|
||||
#endif
|
||||
sink.createDirectory(CanonPath("subdir"));
|
||||
sink.createRegularFile(CanonPath("file1"), [](CreateRegularFileSink & crf) { crf("content1"); });
|
||||
sink.createRegularFile(CanonPath("subdir/file2"), [](CreateRegularFileSink & crf) { crf("content2"); });
|
||||
|
||||
@@ -54,17 +54,6 @@ std::optional<CanonPath> CanonPath::parent() const
|
||||
return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
|
||||
}
|
||||
|
||||
std::optional<std::pair<CanonPath, std::string_view>> CanonPath::parentAndChild() const
|
||||
{
|
||||
if (isRoot())
|
||||
return std::nullopt;
|
||||
auto slash = path.rfind('/');
|
||||
return {{
|
||||
CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, slash))),
|
||||
std::string_view{path}.substr(slash + 1)
|
||||
}};
|
||||
}
|
||||
|
||||
void CanonPath::pop()
|
||||
{
|
||||
assert(!isRoot());
|
||||
|
||||
+16
-166
@@ -115,9 +115,6 @@ Path canonPath(PathView path, bool resolveSymlinks)
|
||||
if (!isAbsolute(path))
|
||||
throw Error("not an absolute path: '%1%'", path);
|
||||
|
||||
// For Windows
|
||||
auto rootName = std::filesystem::path{path}.root_name();
|
||||
|
||||
/* This just exists because we cannot set the target of `remaining`
|
||||
(the callback parameter) directly to a newly-constructed string,
|
||||
since it is `std::string_view`. */
|
||||
@@ -147,8 +144,6 @@ Path canonPath(PathView path, bool resolveSymlinks)
|
||||
}
|
||||
});
|
||||
|
||||
if (!rootName.empty())
|
||||
ret = rootName.string() + std::move(ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -380,14 +375,6 @@ void syncParent(const Path & path)
|
||||
fd.fsync();
|
||||
}
|
||||
|
||||
#ifdef __FreeBSD__
|
||||
# define MOUNTEDPATHS_PARAM , std::set<Path> & mountedPaths
|
||||
# define MOUNTEDPATHS_ARG , mountedPaths
|
||||
#else
|
||||
# define MOUNTEDPATHS_PARAM
|
||||
# define MOUNTEDPATHS_ARG
|
||||
#endif
|
||||
|
||||
void recursiveSync(const Path & path)
|
||||
{
|
||||
/* If it's a file or symlink, just fsync and return. */
|
||||
@@ -432,129 +419,6 @@ void recursiveSync(const Path & path)
|
||||
}
|
||||
}
|
||||
|
||||
static void _deletePath(
|
||||
Descriptor parentfd,
|
||||
const std::filesystem::path & path,
|
||||
uint64_t & bytesFreed,
|
||||
std::exception_ptr & ex MOUNTEDPATHS_PARAM)
|
||||
{
|
||||
#ifndef _WIN32
|
||||
checkInterrupt();
|
||||
|
||||
# ifdef __FreeBSD__
|
||||
// In case of emergency (unmount fails for some reason) not recurse into mountpoints.
|
||||
// This prevents us from tearing up the nullfs-mounted nix store.
|
||||
if (mountedPaths.find(path) != mountedPaths.end()) {
|
||||
return;
|
||||
}
|
||||
# endif
|
||||
|
||||
std::string name(path.filename());
|
||||
assert(name != "." && name != ".." && !name.empty());
|
||||
|
||||
struct stat st;
|
||||
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
|
||||
if (errno == ENOENT)
|
||||
return;
|
||||
throw SysError("getting status of %1%", path);
|
||||
}
|
||||
|
||||
if (!S_ISDIR(st.st_mode)) {
|
||||
/* We are about to delete a file. Will it likely free space? */
|
||||
|
||||
switch (st.st_nlink) {
|
||||
/* Yes: last link. */
|
||||
case 1:
|
||||
bytesFreed += st.st_size;
|
||||
break;
|
||||
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
|
||||
was performed. Instead of checking for real let's assume
|
||||
it's an optimised file and space will be freed.
|
||||
|
||||
In worst case we will double count on freed space for files
|
||||
with exactly two hardlinks for unoptimised packages.
|
||||
*/
|
||||
case 2:
|
||||
bytesFreed += st.st_size;
|
||||
break;
|
||||
/* No: 3+ links. */
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
/* Make the directory accessible. */
|
||||
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
|
||||
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
|
||||
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
|
||||
throw SysError("chmod %1%", path);
|
||||
}
|
||||
|
||||
int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
|
||||
if (fd == -1)
|
||||
throw SysError("opening directory %1%", path);
|
||||
AutoCloseDir dir(fdopendir(fd));
|
||||
if (!dir)
|
||||
throw SysError("opening directory %1%", path);
|
||||
|
||||
struct dirent * dirent;
|
||||
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
||||
checkInterrupt();
|
||||
std::string childName = dirent->d_name;
|
||||
if (childName == "." || childName == "..")
|
||||
continue;
|
||||
_deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG);
|
||||
}
|
||||
if (errno)
|
||||
throw SysError("reading directory %1%", path);
|
||||
}
|
||||
|
||||
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
|
||||
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
|
||||
if (errno == ENOENT)
|
||||
return;
|
||||
try {
|
||||
throw SysError("cannot unlink %1%", path);
|
||||
} catch (...) {
|
||||
if (!ex)
|
||||
ex = std::current_exception();
|
||||
else
|
||||
ignoreExceptionExceptInterrupt();
|
||||
}
|
||||
}
|
||||
#else
|
||||
// TODO implement
|
||||
throw UnimplementedError("_deletePath");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM)
|
||||
{
|
||||
assert(path.is_absolute());
|
||||
assert(path.parent_path() != path);
|
||||
|
||||
AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
|
||||
if (!dirfd) {
|
||||
if (errno == ENOENT)
|
||||
return;
|
||||
throw SysError("opening directory %s", path.parent_path());
|
||||
}
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
_deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG);
|
||||
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
}
|
||||
|
||||
void deletePath(const std::filesystem::path & path)
|
||||
{
|
||||
uint64_t dummy;
|
||||
deletePath(path, dummy);
|
||||
}
|
||||
|
||||
void createDir(const Path & path, mode_t mode)
|
||||
{
|
||||
if (mkdir(
|
||||
@@ -577,25 +441,6 @@ void createDirs(const std::filesystem::path & path)
|
||||
}
|
||||
}
|
||||
|
||||
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
|
||||
{
|
||||
// Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
|
||||
#ifdef __FreeBSD__
|
||||
std::set<Path> mountedPaths;
|
||||
struct statfs * mntbuf;
|
||||
int count;
|
||||
if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
|
||||
throw SysError("getmntinfo");
|
||||
}
|
||||
|
||||
for (int i = 0; i < count; i++) {
|
||||
mountedPaths.emplace(mntbuf[i].f_mntonname);
|
||||
}
|
||||
#endif
|
||||
bytesFreed = 0;
|
||||
_deletePath(path, bytesFreed MOUNTEDPATHS_ARG);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
|
||||
AutoDelete::AutoDelete()
|
||||
@@ -672,11 +517,6 @@ void AutoUnmount::cancel()
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
|
||||
std::filesystem::path defaultTempDir()
|
||||
{
|
||||
return getEnvNonEmpty("TMPDIR").value_or("/tmp");
|
||||
}
|
||||
|
||||
std::filesystem::path createTempDir(const std::filesystem::path & tmpRoot, const std::string & prefix, mode_t mode)
|
||||
{
|
||||
while (1) {
|
||||
@@ -713,17 +553,27 @@ AutoCloseFD createAnonymousTempFile()
|
||||
{
|
||||
AutoCloseFD fd;
|
||||
#ifdef O_TMPFILE
|
||||
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR, S_IWUSR | S_IRUSR);
|
||||
if (!fd)
|
||||
throw SysError("creating anonymous temporary file");
|
||||
#else
|
||||
static std::atomic_flag tmpfileUnsupported{};
|
||||
if (!tmpfileUnsupported.test()) /* Try with O_TMPFILE first. */ {
|
||||
/* Use O_EXCL, because the file is never supposed to be linked into filesystem. */
|
||||
fd = ::open(defaultTempDir().c_str(), O_TMPFILE | O_CLOEXEC | O_RDWR | O_EXCL, S_IWUSR | S_IRUSR);
|
||||
if (!fd) {
|
||||
/* Not supported by the filesystem or the kernel. */
|
||||
if (errno == EOPNOTSUPP || errno == EISDIR)
|
||||
tmpfileUnsupported.test_and_set(); /* Set flag and fall through to createTempFile. */
|
||||
else
|
||||
throw SysError("creating anonymous temporary file");
|
||||
} else {
|
||||
return fd; /* Successfully created. */
|
||||
}
|
||||
}
|
||||
#endif
|
||||
auto [fd2, path] = createTempFile("nix-anonymous");
|
||||
if (!fd2)
|
||||
throw SysError("creating temporary file '%s'", path);
|
||||
fd = std::move(fd2);
|
||||
# ifndef _WIN32
|
||||
#ifndef _WIN32
|
||||
unlink(requireCString(path)); /* We only care about the file descriptor. */
|
||||
# endif
|
||||
#endif
|
||||
return fd;
|
||||
}
|
||||
|
||||
+26
-15
@@ -70,7 +70,31 @@ static std::filesystem::path append(const std::filesystem::path & src, const Can
|
||||
return dst;
|
||||
}
|
||||
|
||||
void RestoreSink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
|
||||
#ifndef _WIN32
|
||||
void RestoreSink::createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
|
||||
{
|
||||
if (path.isRoot()) {
|
||||
createDirectory(path);
|
||||
callback(*this, path);
|
||||
return;
|
||||
}
|
||||
|
||||
createDirectory(path);
|
||||
assert(dirFd); // If that's not true the above call must have thrown an exception.
|
||||
|
||||
RestoreSink dirSink{startFsync};
|
||||
dirSink.dstPath = append(dstPath, path);
|
||||
dirSink.dirFd =
|
||||
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
|
||||
if (!dirSink.dirFd)
|
||||
throw SysError("opening directory '%s'", dirSink.dstPath.string());
|
||||
|
||||
callback(dirSink, CanonPath::root);
|
||||
}
|
||||
#endif
|
||||
|
||||
void RestoreSink::createDirectory(const CanonPath & path)
|
||||
{
|
||||
auto p = append(dstPath, path);
|
||||
|
||||
@@ -83,16 +107,6 @@ void RestoreSink::createDirectory(const CanonPath & path, std::optional<Director
|
||||
if (::mkdirat(dirFd.get(), path.rel_c_str(), 0777) == -1)
|
||||
throw SysError("creating directory '%s'", p.string());
|
||||
|
||||
if (callback) {
|
||||
RestoreSink dirSink{startFsync};
|
||||
dirSink.dstPath =
|
||||
unix::openFileEnsureBeneathNoSymlinks(dirFd.get(), path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
|
||||
if (!dirSink.dirFd)
|
||||
throw SysError("opening directory '%s'", dirSink.dstPath.string());
|
||||
|
||||
(*callback)(dirSink, CanonPath::root);
|
||||
}
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
@@ -111,10 +125,7 @@ void RestoreSink::createDirectory(const CanonPath & path, std::optional<Director
|
||||
throw SysError("creating directory '%1%'", p.string());
|
||||
}
|
||||
#endif
|
||||
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
}
|
||||
};
|
||||
|
||||
struct RestoreRegularFile : CreateRegularFileSink
|
||||
{
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/callback.hh"
|
||||
#include "nix/util/ref.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/post.hpp>
|
||||
#include <boost/asio/awaitable.hpp>
|
||||
#include <boost/asio/use_awaitable.hpp>
|
||||
#include <boost/asio/associated_cancellation_slot.hpp>
|
||||
|
||||
#include <concepts>
|
||||
|
||||
namespace nix {
|
||||
|
||||
namespace asio = boost::asio;
|
||||
|
||||
template<typename T, std::invocable<Callback<T>> F, typename CompletionToken>
|
||||
auto callbackToAwaitable(F && initiate, CompletionToken && token)
|
||||
{
|
||||
return asio::async_initiate<CompletionToken, void(std::future<T>)>(
|
||||
[initiate = std::forward<F>(initiate)](auto handler) mutable {
|
||||
auto executor = asio::get_associated_executor(handler);
|
||||
auto done = std::make_shared<std::atomic<bool>>(false);
|
||||
auto h = std::make_shared<decltype(handler)>(std::move(handler));
|
||||
|
||||
if (auto slot = asio::get_associated_cancellation_slot(*h); slot.is_connected()) {
|
||||
std::weak_ptr wh = h; /* To handle the cyclic ownership. */
|
||||
std::weak_ptr wdone = done;
|
||||
slot.assign([executor, wh, wdone](asio::cancellation_type /*don't care*/) {
|
||||
auto h = wh.lock();
|
||||
auto done = wdone.lock();
|
||||
if (!h || !done || done->exchange(true))
|
||||
return; /* Gracefully die. */
|
||||
/* Doesn't need to be kept alive for get_future() since it shares the ownership. */
|
||||
std::promise<T> p;
|
||||
p.set_exception(std::make_exception_ptr(Interrupted("interrupted by user")));
|
||||
asio::post(executor, [h, fut = p.get_future()]() mutable { std::move (*h)(std::move(fut)); });
|
||||
});
|
||||
}
|
||||
|
||||
initiate(Callback<T>([executor, done, h](std::future<T> fut) mutable {
|
||||
if (done->exchange(true))
|
||||
/* Early return for cooperative cancellation. The callback has been caller
|
||||
later than we've been cancelled. In practice we'll get an error, the handler
|
||||
has already been posted by the cancellation handler. */
|
||||
return;
|
||||
asio::post(executor, [h, fut = std::move(fut)]() mutable { std::move (*h)(std::move(fut)); });
|
||||
}));
|
||||
},
|
||||
std::forward<CompletionToken>(token));
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a completion handler callback into a stackless coroutine. The
|
||||
* callback can be invoked on any thread and the completion handler will be
|
||||
* marshalled to the coroutines executer.
|
||||
*/
|
||||
template<typename T, std::invocable<Callback<T>> F>
|
||||
asio::awaitable<T> callbackToAwaitable(F && initiate)
|
||||
{
|
||||
auto fut = co_await callbackToAwaitable<T>(std::forward<F>(initiate), asio::use_awaitable);
|
||||
co_return fut.get();
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -238,13 +238,6 @@ public:
|
||||
return ((std::string_view) path).substr(path.rfind('/') + 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Combination of `parent` and `baseName`.
|
||||
*
|
||||
* @note that because of the `std::string_view`, we are borrowing for the base name.
|
||||
*/
|
||||
std::optional<std::pair<CanonPath, std::string_view>> parentAndChild() const;
|
||||
|
||||
bool operator==(const CanonPath & x) const
|
||||
{
|
||||
return path == x.path;
|
||||
|
||||
@@ -1,73 +1,167 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include <set>
|
||||
#include <future>
|
||||
#include "nix/util/sync.hh"
|
||||
#include "nix/util/async.hh"
|
||||
#include "nix/util/ref.hh"
|
||||
|
||||
using std::set;
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/post.hpp>
|
||||
#include <boost/asio/strand.hpp>
|
||||
#include <boost/asio/detached.hpp>
|
||||
#include <boost/asio/awaitable.hpp>
|
||||
#include <boost/asio/co_spawn.hpp>
|
||||
#include <boost/asio/bind_executor.hpp>
|
||||
#include <boost/asio/executor_work_guard.hpp>
|
||||
|
||||
#include <queue>
|
||||
#include <set>
|
||||
|
||||
namespace nix {
|
||||
|
||||
template<typename T>
|
||||
using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
|
||||
using GetEdgesAsync = std::function<asio::awaitable<std::set<T>>(const T & elt)>;
|
||||
|
||||
template<typename T>
|
||||
void computeClosure(const set<T> startElts, set<T> & res, GetEdgesAsync<T> getEdgesAsync)
|
||||
template<typename T, typename CompletionToken>
|
||||
auto computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges, CompletionToken && token)
|
||||
{
|
||||
struct State
|
||||
{
|
||||
size_t pending;
|
||||
set<T> & res;
|
||||
std::exception_ptr exc;
|
||||
};
|
||||
auto initiator = [&res, startElts = std::move(startElts), getEdges = std::move(getEdges)](auto handler) {
|
||||
auto executor = asio::make_strand(asio::get_associated_executor(handler));
|
||||
|
||||
Sync<State> state_(State{0, res, 0});
|
||||
|
||||
std::condition_variable done;
|
||||
|
||||
auto enqueue = [&](this auto & enqueue, const T & current) -> void {
|
||||
/* Hand-rolled dynamic async graph traversal. ASIO/Cobalt and standard
|
||||
* C++ will get channels and task groups at some point, but this will
|
||||
* have to suffice for now. */
|
||||
struct State : std::enable_shared_from_this<State>
|
||||
{
|
||||
auto state(state_.lock());
|
||||
if (state->exc)
|
||||
return;
|
||||
if (!state->res.insert(current).second)
|
||||
return;
|
||||
state->pending++;
|
||||
decltype(executor) executor;
|
||||
decltype(getEdges) getEdges;
|
||||
decltype(handler) handler;
|
||||
std::set<T> & res;
|
||||
/**
|
||||
* Needed to keep the ctx.run() alive because actual work might be happening on another thread
|
||||
* (like with FileTransfer case).
|
||||
*/
|
||||
asio::executor_work_guard<decltype(State::executor)> workGuard;
|
||||
std::size_t pending = 0;
|
||||
/**
|
||||
* Whether the completion handler has been called.
|
||||
*/
|
||||
bool done = false;
|
||||
/**
|
||||
* Amount of coroutines currently in flight.
|
||||
*/
|
||||
std::size_t inFlight = 0;
|
||||
/**
|
||||
* Maximum number of concurrent coroutines. Implements primitive rate limiting.
|
||||
*/
|
||||
std::size_t maxConcurrent = 64;
|
||||
/**
|
||||
* Nodes to handle next.
|
||||
*/
|
||||
std::queue<T> todo;
|
||||
|
||||
State(
|
||||
decltype(executor) executor_, decltype(getEdges) getEdges, decltype(handler) handler, std::set<T> & res)
|
||||
: executor(executor_)
|
||||
, getEdges(std::move(getEdges))
|
||||
, handler(std::move(handler))
|
||||
, res(res)
|
||||
, workGuard(asio::make_work_guard(executor_))
|
||||
{
|
||||
}
|
||||
|
||||
void complete(std::exception_ptr ex)
|
||||
{
|
||||
if (std::exchange(done, true))
|
||||
return;
|
||||
workGuard.reset(); /* We are done and we can release the lock. */
|
||||
asio::post(executor, [state = this->shared_from_this(), ex] { state->handler(ex); });
|
||||
}
|
||||
|
||||
void enqueue(const std::set<T> & elts)
|
||||
{
|
||||
for (const auto & elt : elts)
|
||||
enqueue(elt);
|
||||
}
|
||||
|
||||
void spawnWorker(const T & elt)
|
||||
{
|
||||
++inFlight;
|
||||
auto state = this->shared_from_this();
|
||||
asio::post(executor, [state = this->shared_from_this(), elt = std::move(elt)] {
|
||||
asio::co_spawn(
|
||||
state->executor,
|
||||
[state, elt]() -> asio::awaitable<void> {
|
||||
try {
|
||||
state->enqueue(co_await state->getEdges(elt));
|
||||
} catch (...) {
|
||||
state->complete(std::current_exception());
|
||||
}
|
||||
state->onWorkDone();
|
||||
},
|
||||
asio::detached);
|
||||
});
|
||||
}
|
||||
|
||||
void onWorkDone()
|
||||
{
|
||||
--inFlight;
|
||||
--pending;
|
||||
|
||||
if (!todo.empty()) {
|
||||
auto next = std::move(todo.front());
|
||||
todo.pop();
|
||||
asio::post(executor, [state = this->shared_from_this(), next = std::move(next)]() mutable {
|
||||
state->spawnWorker(std::move(next));
|
||||
});
|
||||
} else if (pending == 0) {
|
||||
complete(std::exception_ptr{});
|
||||
}
|
||||
}
|
||||
|
||||
void enqueue(const T & elt)
|
||||
{
|
||||
if (done)
|
||||
return;
|
||||
|
||||
if (!res.insert(elt).second)
|
||||
return;
|
||||
|
||||
++pending;
|
||||
|
||||
if (inFlight < maxConcurrent) {
|
||||
spawnWorker(elt);
|
||||
} else {
|
||||
todo.push(elt);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
auto state = make_ref<State>(executor, std::move(getEdges), std::move(handler), res);
|
||||
if (startElts.empty()) {
|
||||
/* No work to do. */
|
||||
state->complete(std::exception_ptr{});
|
||||
return;
|
||||
}
|
||||
|
||||
getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
|
||||
try {
|
||||
auto children = prom.get_future().get();
|
||||
for (auto & child : children)
|
||||
enqueue(child);
|
||||
{
|
||||
auto state(state_.lock());
|
||||
assert(state->pending);
|
||||
if (!--state->pending)
|
||||
done.notify_one();
|
||||
}
|
||||
} catch (...) {
|
||||
auto state(state_.lock());
|
||||
if (!state->exc)
|
||||
state->exc = std::current_exception();
|
||||
assert(state->pending);
|
||||
if (!--state->pending)
|
||||
done.notify_one();
|
||||
};
|
||||
});
|
||||
asio::post(executor, [state, startElts = std::move(startElts)] { state->enqueue(startElts); });
|
||||
};
|
||||
|
||||
for (auto & startElt : startElts)
|
||||
enqueue(startElt);
|
||||
return asio::async_initiate<CompletionToken, void(std::exception_ptr)>(std::move(initiator), token);
|
||||
}
|
||||
|
||||
{
|
||||
auto state(state_.lock());
|
||||
while (state->pending)
|
||||
state.wait(done);
|
||||
if (state->exc)
|
||||
std::rethrow_exception(state->exc);
|
||||
}
|
||||
template<typename T>
|
||||
void computeClosure(std::set<T> startElts, std::set<T> & res, GetEdgesAsync<T> getEdges)
|
||||
{
|
||||
asio::io_context ctx;
|
||||
std::exception_ptr ex = nullptr;
|
||||
computeClosure(
|
||||
std::move(startElts),
|
||||
res,
|
||||
std::move(getEdges),
|
||||
asio::bind_executor(ctx.get_executor(), [&](std::exception_ptr ex2) { ex = ex2; }));
|
||||
ctx.run();
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -261,4 +261,14 @@ Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & p
|
||||
|
||||
MakeError(EndOfFile, Error);
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
/**
|
||||
* Windows specific replacement for POSIX `lseek` that operates on a `HANDLE` and not
|
||||
* a file descriptor.
|
||||
*/
|
||||
off_t lseek(Descriptor fd, off_t offset, int whence);
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace nix
|
||||
@@ -40,6 +40,11 @@ struct UnixPathTrait
|
||||
{
|
||||
return path.rfind('/', from);
|
||||
}
|
||||
|
||||
static size_t rootNameLen(StringView)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -83,6 +88,18 @@ struct WindowsPathTrait
|
||||
size_t p2 = path.rfind(preferredSep, from);
|
||||
return p1 == String::npos ? p2 : p2 == String::npos ? p1 : std::max(p1, p2);
|
||||
}
|
||||
|
||||
static size_t rootNameLen(StringView path)
|
||||
{
|
||||
if (path.size() >= 2 && path[1] == ':') {
|
||||
char driveLetter = path[0];
|
||||
if ((driveLetter >= 'A' && driveLetter <= 'Z') || (driveLetter >= 'a' && driveLetter <= 'z'))
|
||||
return 2;
|
||||
}
|
||||
/* TODO: This needs to also handle UNC paths.
|
||||
* https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats#unc-paths */
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
template<typename CharT>
|
||||
@@ -116,6 +133,11 @@ typename PathDict::String canonPathInner(typename PathDict::StringView remaining
|
||||
typename PathDict::String result;
|
||||
result.reserve(256);
|
||||
|
||||
if (auto rootNameLength = PathDict::rootNameLen(remaining)) {
|
||||
result += remaining.substr(0, rootNameLength); /* Copy drive letter verbatim. */
|
||||
remaining.remove_prefix(rootNameLength);
|
||||
}
|
||||
|
||||
while (true) {
|
||||
|
||||
/* Skip slashes. */
|
||||
|
||||
@@ -362,6 +362,8 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
|
||||
|
||||
/**
|
||||
* Return `TMPDIR`, or the default temporary directory if unset or empty.
|
||||
* Uses GetTempPathW on windows which respects TMP, TEMP, USERPROFILE env variables.
|
||||
* Does not resolve symlinks and the returned path might not be directory or exist at all.
|
||||
*/
|
||||
std::filesystem::path defaultTempDir();
|
||||
|
||||
|
||||
@@ -34,6 +34,8 @@ struct FileSystemObjectSink
|
||||
{
|
||||
virtual ~FileSystemObjectSink() = default;
|
||||
|
||||
virtual void createDirectory(const CanonPath & path) = 0;
|
||||
|
||||
using DirectoryCreatedCallback = std::function<void(FileSystemObjectSink & dirSink, const CanonPath & dirRelPath)>;
|
||||
|
||||
/**
|
||||
@@ -45,7 +47,11 @@ struct FileSystemObjectSink
|
||||
* freshly created directory. Use this when it's important to disallow any
|
||||
* intermediate path components from being symlinks.
|
||||
*/
|
||||
virtual void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) = 0;
|
||||
virtual void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback)
|
||||
{
|
||||
createDirectory(path);
|
||||
callback(*this, path);
|
||||
}
|
||||
|
||||
/**
|
||||
* This function in general is no re-entrant. Only one file can be
|
||||
@@ -80,11 +86,7 @@ void copyRecursive(
|
||||
*/
|
||||
struct NullFileSystemObjectSink : FileSystemObjectSink
|
||||
{
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
{
|
||||
if (callback)
|
||||
(*callback)(*this, CanonPath::root);
|
||||
}
|
||||
void createDirectory(const CanonPath & path) override {}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override {}
|
||||
|
||||
@@ -116,7 +118,11 @@ struct RestoreSink : FileSystemObjectSink
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
|
||||
void createDirectory(const CanonPath & path) override;
|
||||
|
||||
#ifndef _WIN32
|
||||
void createDirectory(const CanonPath & path, DirectoryCreatedCallback callback) override;
|
||||
#endif
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
|
||||
|
||||
@@ -138,15 +144,9 @@ struct RegularFileSink : FileSystemObjectSink
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
regular = false;
|
||||
if (callback) {
|
||||
NullFileSystemObjectSink s;
|
||||
/* The children of the
|
||||
directory that cannot exist also cannot exist */
|
||||
(*callback)(s, CanonPath::root);
|
||||
}
|
||||
}
|
||||
|
||||
void createSymlink(const CanonPath & path, const std::string & target) override
|
||||
|
||||
@@ -153,7 +153,7 @@ struct MemorySink : FileSystemObjectSink
|
||||
{
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override;
|
||||
void createDirectory(const CanonPath & path) override;
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)>) override;
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ headers = files(
|
||||
'args.hh',
|
||||
'args/root.hh',
|
||||
'array-from-string-literal.hh',
|
||||
'async.hh',
|
||||
'base-n.hh',
|
||||
'base-nix-32.hh',
|
||||
'callback.hh',
|
||||
@@ -65,6 +66,7 @@ headers = files(
|
||||
'signals.hh',
|
||||
'signature/local-keys.hh',
|
||||
'signature/signer.hh',
|
||||
'socket.hh',
|
||||
'sort.hh',
|
||||
'source-accessor.hh',
|
||||
'source-path.hh',
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
#pragma once
|
||||
///@file
|
||||
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
|
||||
#ifdef _WIN32
|
||||
# include <winsock2.h>
|
||||
#endif
|
||||
|
||||
namespace nix {
|
||||
|
||||
/**
|
||||
* Often we want to use `Descriptor`, but Windows makes a slightly
|
||||
* stronger file descriptor vs socket distinction, at least at the level
|
||||
* of C types.
|
||||
*/
|
||||
using Socket =
|
||||
#ifdef _WIN32
|
||||
SOCKET
|
||||
#else
|
||||
int
|
||||
#endif
|
||||
;
|
||||
|
||||
#ifdef _WIN32
|
||||
/**
|
||||
* Windows gives this a different name
|
||||
*/
|
||||
# define SHUT_WR SD_SEND
|
||||
# define SHUT_RDWR SD_BOTH
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Convert a `Descriptor` to a `Socket`
|
||||
*
|
||||
* This is a no-op except on Windows.
|
||||
*/
|
||||
static inline Socket toSocket(Descriptor fd)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return reinterpret_cast<Socket>(fd);
|
||||
#else
|
||||
return fd;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a `Socket` to a `Descriptor`
|
||||
*
|
||||
* This is a no-op except on Windows.
|
||||
*/
|
||||
static inline Descriptor fromSocket(Socket fd)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return reinterpret_cast<Descriptor>(fd);
|
||||
#else
|
||||
return fd;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -3,10 +3,8 @@
|
||||
|
||||
#include "nix/util/types.hh"
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
#include "nix/util/socket.hh"
|
||||
|
||||
#ifdef _WIN32
|
||||
# include <winsock2.h>
|
||||
#endif
|
||||
#include <unistd.h>
|
||||
|
||||
#include <filesystem>
|
||||
@@ -23,55 +21,6 @@ AutoCloseFD createUnixDomainSocket();
|
||||
*/
|
||||
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
|
||||
|
||||
/**
|
||||
* Often we want to use `Descriptor`, but Windows makes a slightly
|
||||
* stronger file descriptor vs socket distinction, at least at the level
|
||||
* of C types.
|
||||
*/
|
||||
using Socket =
|
||||
#ifdef _WIN32
|
||||
SOCKET
|
||||
#else
|
||||
int
|
||||
#endif
|
||||
;
|
||||
|
||||
#ifdef _WIN32
|
||||
/**
|
||||
* Windows gives this a different name
|
||||
*/
|
||||
# define SHUT_WR SD_SEND
|
||||
# define SHUT_RDWR SD_BOTH
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Convert a `Socket` to a `Descriptor`
|
||||
*
|
||||
* This is a no-op except on Windows.
|
||||
*/
|
||||
static inline Socket toSocket(Descriptor fd)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return reinterpret_cast<Socket>(fd);
|
||||
#else
|
||||
return fd;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a `Socket` to a `Descriptor`
|
||||
*
|
||||
* This is a no-op except on Windows.
|
||||
*/
|
||||
static inline Descriptor fromSocket(Socket fd)
|
||||
{
|
||||
#ifdef _WIN32
|
||||
return reinterpret_cast<Descriptor>(fd);
|
||||
#else
|
||||
return fd;
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind a Unix domain socket to a path.
|
||||
*/
|
||||
|
||||
@@ -146,7 +146,7 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents
|
||||
|
||||
using File = MemorySourceAccessor::File;
|
||||
|
||||
void MemorySink::createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback)
|
||||
void MemorySink::createDirectory(const CanonPath & path)
|
||||
{
|
||||
auto * f = dst.open(path, File{File::Directory{}});
|
||||
if (!f)
|
||||
@@ -154,10 +154,7 @@ void MemorySink::createDirectory(const CanonPath & path, std::optional<Directory
|
||||
|
||||
if (!std::holds_alternative<File::Directory>(f->raw))
|
||||
throw Error("file '%s' is not a directory", path);
|
||||
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
}
|
||||
};
|
||||
|
||||
struct CreateMemoryRegularFile : CreateRegularFileSink
|
||||
{
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "nix/util/nar-accessor.hh"
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
#include "nix/util/archive.hh"
|
||||
|
||||
#include <map>
|
||||
@@ -98,13 +99,11 @@ struct NarAccessor : public SourceAccessor
|
||||
}
|
||||
}
|
||||
|
||||
void createDirectory(const CanonPath & path, std::optional<DirectoryCreatedCallback> callback = {}) override
|
||||
void createDirectory(const CanonPath & path) override
|
||||
{
|
||||
createMember(
|
||||
path,
|
||||
NarMember{.stat = {.type = Type::tDirectory, .fileSize = 0, .isExecutable = false, .narOffset = 0}});
|
||||
if (callback)
|
||||
(*callback)(*this, path);
|
||||
}
|
||||
|
||||
void createRegularFile(const CanonPath & path, std::function<void(CreateRegularFileSink &)> func) override
|
||||
@@ -283,7 +282,7 @@ GetNarBytes seekableGetNarBytes(const Path & path)
|
||||
GetNarBytes seekableGetNarBytes(Descriptor fd)
|
||||
{
|
||||
return [fd](uint64_t offset, uint64_t length) {
|
||||
if (::lseek(fromDescriptorReadOnly(fd), offset, SEEK_SET) == -1)
|
||||
if (lseek(fd, offset, SEEK_SET) == -1)
|
||||
throw SysError("seeking in file");
|
||||
|
||||
std::string buf(length, 0);
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#include "nix/util/serialise.hh"
|
||||
#include "nix/util/file-descriptor.hh"
|
||||
#include "nix/util/compression.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/socket.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include <cstring>
|
||||
@@ -11,7 +13,6 @@
|
||||
|
||||
#ifdef _WIN32
|
||||
# include <fileapi.h>
|
||||
# include <winsock2.h>
|
||||
# include "nix/util/windows-error.hh"
|
||||
#else
|
||||
# include <poll.h>
|
||||
@@ -162,11 +163,11 @@ size_t FdSource::readUnbuffered(char * data, size_t len)
|
||||
_good = false;
|
||||
throw SysError("reading from file");
|
||||
}
|
||||
#endif
|
||||
if (n == 0) {
|
||||
_good = false;
|
||||
throw EndOfFile(std::string(*endOfFileError));
|
||||
}
|
||||
#endif
|
||||
read += n;
|
||||
return n;
|
||||
}
|
||||
@@ -184,20 +185,20 @@ bool FdSource::hasData()
|
||||
while (true) {
|
||||
fd_set fds;
|
||||
FD_ZERO(&fds);
|
||||
int fd_ = fromDescriptorReadOnly(fd);
|
||||
FD_SET(fd_, &fds);
|
||||
Socket sock = toSocket(fd);
|
||||
FD_SET(sock, &fds);
|
||||
|
||||
struct timeval timeout;
|
||||
timeout.tv_sec = 0;
|
||||
timeout.tv_usec = 0;
|
||||
|
||||
auto n = select(fd_ + 1, &fds, nullptr, nullptr, &timeout);
|
||||
auto n = select(sock + 1, &fds, nullptr, nullptr, &timeout);
|
||||
if (n < 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
throw SysError("polling file descriptor");
|
||||
}
|
||||
return FD_ISSET(fd, &fds);
|
||||
return FD_ISSET(sock, &fds);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,8 +208,7 @@ void FdSource::restart()
|
||||
throw Error("can't seek to the start of a file");
|
||||
buffer.reset();
|
||||
read = bufPosIn = bufPosOut = 0;
|
||||
int fd_ = fromDescriptorReadOnly(fd);
|
||||
if (lseek(fd_, 0, SEEK_SET) == -1)
|
||||
if (lseek(fd, 0, SEEK_SET) == -1)
|
||||
throw SysError("seeking to the start of a file");
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
#include <unistd.h>
|
||||
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/environment-variables.hh"
|
||||
#include "nix/util/signals.hh"
|
||||
#include "nix/util/util.hh"
|
||||
|
||||
#include "util-unix-config-private.hh"
|
||||
|
||||
@@ -19,6 +22,11 @@ Descriptor openDirectory(const std::filesystem::path & path)
|
||||
return open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||
}
|
||||
|
||||
std::filesystem::path defaultTempDir()
|
||||
{
|
||||
return getEnvNonEmpty("TMPDIR").value_or("/tmp");
|
||||
}
|
||||
|
||||
void setWriteTime(
|
||||
const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional<bool> optIsSymlink)
|
||||
{
|
||||
@@ -66,4 +74,154 @@ void setWriteTime(
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef __FreeBSD__
|
||||
# define MOUNTEDPATHS_PARAM , std::set<Path> & mountedPaths
|
||||
# define MOUNTEDPATHS_ARG , mountedPaths
|
||||
#else
|
||||
# define MOUNTEDPATHS_PARAM
|
||||
# define MOUNTEDPATHS_ARG
|
||||
#endif
|
||||
|
||||
static void _deletePath(
|
||||
Descriptor parentfd,
|
||||
const std::filesystem::path & path,
|
||||
uint64_t & bytesFreed,
|
||||
std::exception_ptr & ex MOUNTEDPATHS_PARAM)
|
||||
{
|
||||
#ifndef _WIN32
|
||||
checkInterrupt();
|
||||
|
||||
# ifdef __FreeBSD__
|
||||
// In case of emergency (unmount fails for some reason) not recurse into mountpoints.
|
||||
// This prevents us from tearing up the nullfs-mounted nix store.
|
||||
if (mountedPaths.find(path) != mountedPaths.end()) {
|
||||
return;
|
||||
}
|
||||
# endif
|
||||
|
||||
std::string name(path.filename());
|
||||
assert(name != "." && name != ".." && !name.empty());
|
||||
|
||||
struct stat st;
|
||||
if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
|
||||
if (errno == ENOENT)
|
||||
return;
|
||||
throw SysError("getting status of %1%", path);
|
||||
}
|
||||
|
||||
if (!S_ISDIR(st.st_mode)) {
|
||||
/* We are about to delete a file. Will it likely free space? */
|
||||
|
||||
switch (st.st_nlink) {
|
||||
/* Yes: last link. */
|
||||
case 1:
|
||||
bytesFreed += st.st_size;
|
||||
break;
|
||||
/* Maybe: yes, if 'auto-optimise-store' or manual optimisation
|
||||
was performed. Instead of checking for real let's assume
|
||||
it's an optimised file and space will be freed.
|
||||
|
||||
In worst case we will double count on freed space for files
|
||||
with exactly two hardlinks for unoptimised packages.
|
||||
*/
|
||||
case 2:
|
||||
bytesFreed += st.st_size;
|
||||
break;
|
||||
/* No: 3+ links. */
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
/* Make the directory accessible. */
|
||||
const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
|
||||
if ((st.st_mode & PERM_MASK) != PERM_MASK) {
|
||||
if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
|
||||
throw SysError("chmod %1%", path);
|
||||
}
|
||||
|
||||
int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
|
||||
if (fd == -1)
|
||||
throw SysError("opening directory %1%", path);
|
||||
AutoCloseDir dir(fdopendir(fd));
|
||||
if (!dir)
|
||||
throw SysError("opening directory %1%", path);
|
||||
|
||||
struct dirent * dirent;
|
||||
while (errno = 0, dirent = readdir(dir.get())) { /* sic */
|
||||
checkInterrupt();
|
||||
std::string childName = dirent->d_name;
|
||||
if (childName == "." || childName == "..")
|
||||
continue;
|
||||
_deletePath(dirfd(dir.get()), path / childName, bytesFreed, ex MOUNTEDPATHS_ARG);
|
||||
}
|
||||
if (errno)
|
||||
throw SysError("reading directory %1%", path);
|
||||
}
|
||||
|
||||
int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
|
||||
if (unlinkat(parentfd, name.c_str(), flags) == -1) {
|
||||
if (errno == ENOENT)
|
||||
return;
|
||||
try {
|
||||
throw SysError("cannot unlink %1%", path);
|
||||
} catch (...) {
|
||||
if (!ex)
|
||||
ex = std::current_exception();
|
||||
else
|
||||
ignoreExceptionExceptInterrupt();
|
||||
}
|
||||
}
|
||||
#else
|
||||
// TODO implement
|
||||
throw UnimplementedError("_deletePath");
|
||||
#endif
|
||||
}
|
||||
|
||||
static void _deletePath(const std::filesystem::path & path, uint64_t & bytesFreed MOUNTEDPATHS_PARAM)
|
||||
{
|
||||
assert(path.is_absolute());
|
||||
assert(path.parent_path() != path);
|
||||
|
||||
AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
|
||||
if (!dirfd) {
|
||||
if (errno == ENOENT)
|
||||
return;
|
||||
throw SysError("opening directory %s", path.parent_path());
|
||||
}
|
||||
|
||||
std::exception_ptr ex;
|
||||
|
||||
_deletePath(dirfd.get(), path, bytesFreed, ex MOUNTEDPATHS_ARG);
|
||||
|
||||
if (ex)
|
||||
std::rethrow_exception(ex);
|
||||
}
|
||||
|
||||
void deletePath(const std::filesystem::path & path)
|
||||
{
|
||||
uint64_t dummy;
|
||||
deletePath(path, dummy);
|
||||
}
|
||||
|
||||
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
|
||||
{
|
||||
// Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path);
|
||||
#ifdef __FreeBSD__
|
||||
std::set<Path> mountedPaths;
|
||||
struct statfs * mntbuf;
|
||||
int count;
|
||||
if ((count = getmntinfo(&mntbuf, MNT_WAIT)) < 0) {
|
||||
throw SysError("getmntinfo");
|
||||
}
|
||||
|
||||
for (int i = 0; i < count; i++) {
|
||||
mountedPaths.emplace(mntbuf[i].f_mntonname);
|
||||
}
|
||||
#endif
|
||||
bytesFreed = 0;
|
||||
_deletePath(path, bytesFreed MOUNTEDPATHS_ARG);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
@@ -5,13 +5,12 @@
|
||||
#include "nix/util/windows-error.hh"
|
||||
#include "nix/util/file-path.hh"
|
||||
|
||||
#ifdef _WIN32
|
||||
# include <fileapi.h>
|
||||
# include <error.h>
|
||||
# include <namedpipeapi.h>
|
||||
# include <namedpipeapi.h>
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
# include <windows.h>
|
||||
#include <fileapi.h>
|
||||
#include <error.h>
|
||||
#include <namedpipeapi.h>
|
||||
#include <namedpipeapi.h>
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
|
||||
namespace nix {
|
||||
|
||||
@@ -46,16 +45,16 @@ void writeFull(HANDLE handle, std::string_view s, bool allowInterrupts)
|
||||
if (allowInterrupts)
|
||||
checkInterrupt();
|
||||
DWORD res;
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
#if _WIN32_WINNT >= 0x0600
|
||||
auto path = handleToPath(handle); // debug; do it before because handleToPath changes lasterror
|
||||
if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) {
|
||||
throw WinError("writing to file %1%:%2%", handle, path);
|
||||
}
|
||||
# else
|
||||
#else
|
||||
if (!WriteFile(handle, s.data(), s.size(), &res, NULL)) {
|
||||
throw WinError("writing to file %1%", handle);
|
||||
}
|
||||
# endif
|
||||
#endif
|
||||
if (res > 0)
|
||||
s.remove_prefix(res);
|
||||
}
|
||||
@@ -120,7 +119,7 @@ void Pipe::create()
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
|
||||
# if _WIN32_WINNT >= 0x0600
|
||||
#if _WIN32_WINNT >= 0x0600
|
||||
|
||||
std::wstring windows::handleToFileName(HANDLE handle)
|
||||
{
|
||||
@@ -149,7 +148,37 @@ Path windows::handleToPath(HANDLE handle)
|
||||
return os_string_to_string(handleToFileName(handle));
|
||||
}
|
||||
|
||||
# endif
|
||||
#endif
|
||||
|
||||
off_t lseek(HANDLE h, off_t offset, int whence)
|
||||
{
|
||||
DWORD method;
|
||||
switch (whence) {
|
||||
case SEEK_SET:
|
||||
method = FILE_BEGIN;
|
||||
break;
|
||||
case SEEK_CUR:
|
||||
method = FILE_CURRENT;
|
||||
break;
|
||||
case SEEK_END:
|
||||
method = FILE_END;
|
||||
break;
|
||||
default:
|
||||
throw Error("lseek: invalid whence %d", whence);
|
||||
}
|
||||
|
||||
LARGE_INTEGER li;
|
||||
li.QuadPart = offset;
|
||||
LARGE_INTEGER newPos;
|
||||
|
||||
if (!SetFilePointerEx(h, li, &newPos, method)) {
|
||||
/* Convert to a POSIX error, since caller code works with this as if it were
|
||||
a POSIX lseek. */
|
||||
errno = std::error_code(GetLastError(), std::system_category()).default_error_condition().value();
|
||||
return -1;
|
||||
}
|
||||
|
||||
return newPos.QuadPart;
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
#endif
|
||||
@@ -1,9 +1,11 @@
|
||||
#include "nix/util/file-system.hh"
|
||||
#include "nix/util/windows-error.hh"
|
||||
#include "nix/util/logging.hh"
|
||||
|
||||
#ifdef _WIN32
|
||||
namespace nix {
|
||||
|
||||
using namespace nix::windows;
|
||||
|
||||
void setWriteTime(
|
||||
const std::filesystem::path & path, time_t accessedTime, time_t modificationTime, std::optional<bool> optIsSymlink)
|
||||
{
|
||||
@@ -28,5 +30,27 @@ Descriptor openDirectory(const std::filesystem::path & path)
|
||||
NULL);
|
||||
}
|
||||
|
||||
std::filesystem::path defaultTempDir()
|
||||
{
|
||||
wchar_t buf[MAX_PATH + 1];
|
||||
DWORD len = GetTempPathW(MAX_PATH + 1, buf);
|
||||
if (len == 0 || len > MAX_PATH)
|
||||
throw WinError("getting default temporary directory");
|
||||
return std::filesystem::path(buf);
|
||||
}
|
||||
|
||||
void deletePath(const std::filesystem::path & path)
|
||||
{
|
||||
std::error_code ec;
|
||||
std::filesystem::remove_all(path, ec);
|
||||
if (ec && ec != std::errc::no_such_file_or_directory)
|
||||
throw SysError(ec.default_error_condition().value(), "recursively deleting %1%", path);
|
||||
}
|
||||
|
||||
void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed)
|
||||
{
|
||||
bytesFreed = 0;
|
||||
deletePath(path);
|
||||
}
|
||||
|
||||
} // namespace nix
|
||||
#endif
|
||||
@@ -437,22 +437,23 @@ static void forwardStdioConnection(RemoteStore & store)
|
||||
int from = conn->from.fd;
|
||||
int to = conn->to.fd;
|
||||
|
||||
auto nfds = std::max(from, STDIN_FILENO) + 1;
|
||||
Socket fromSock = toSocket(from), stdinSock = toSocket(getStandardInput());
|
||||
auto nfds = std::max(fromSock, stdinSock) + 1;
|
||||
while (true) {
|
||||
fd_set fds;
|
||||
FD_ZERO(&fds);
|
||||
FD_SET(from, &fds);
|
||||
FD_SET(STDIN_FILENO, &fds);
|
||||
FD_SET(fromSock, &fds);
|
||||
FD_SET(stdinSock, &fds);
|
||||
if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
|
||||
throw SysError("waiting for data from client or server");
|
||||
if (FD_ISSET(from, &fds)) {
|
||||
if (FD_ISSET(fromSock, &fds)) {
|
||||
auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
|
||||
if (res == -1)
|
||||
throw SysError("splicing data from daemon socket to stdout");
|
||||
else if (res == 0)
|
||||
throw EndOfFile("unexpected EOF from daemon socket");
|
||||
}
|
||||
if (FD_ISSET(STDIN_FILENO, &fds)) {
|
||||
if (FD_ISSET(stdinSock, &fds)) {
|
||||
auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
|
||||
if (res == -1)
|
||||
throw SysError("splicing data from stdin to daemon socket");
|
||||
|
||||
Reference in new issue
Block a user