Compare commits

...

18 Commits

Author SHA1 Message Date
Sergei Zimmerman
545f5fb309 libstore: Use race-free copyRecursive with makeFSSourceAccessor and RestoreSink when copying FOD outputs 2026-01-10 03:10:46 +03:00
Sergei Zimmerman
6312a94839 libutil: Add a new overload of readDirectory (for fd-relative operations) and use in recursive traversal
This ensures race-free traversal throughout.
2026-01-10 03:10:45 +03:00
Sergei Zimmerman
e77306792d libstore: Use requireStoreObjectAccessor in addToStore 2026-01-10 03:10:39 +03:00
Sergei Zimmerman
24054b9c5d libstore: LocalStoreAccessor uses makeFSSourceAccessor
This more honestly wraps the underlying FS accessor (we want to use
the unix-specific dirfd-based one).
2026-01-10 03:10:38 +03:00
Sergei Zimmerman
404cba4347 libutil: Get rid of PosixSourceAccessor::createAtRoot 2026-01-10 03:10:37 +03:00
Sergei Zimmerman
63aa29c2be nix-perl: Get rid of the last occurence of createAtRoot 2026-01-10 03:10:36 +03:00
Sergei Zimmerman
3245211b29 libutil: Use makeFSSourceAccessor in dumpPath
Unlike previously, we now follow symlinks in parents (not the last path component),
but this is secure and what versions like 2.18 did.
2026-01-10 03:10:36 +03:00
Sergei Zimmerman
ebb4271c6f nix: Use makeFSSourceAccessor in place of createAtRoot
The API is the same and uses the unix-specific implementation.
2026-01-10 03:10:35 +03:00
Sergei Zimmerman
af854ece5f libcmd: Use makeFSSourceAccessor for reference-lock-file cli option 2026-01-10 03:10:34 +03:00
Sergei Zimmerman
803f864424 libfetchers: Use source accessor if the mercurial fetcher, use makeFSSourceAccessor
This significantly simplifies path filtering and gets rid of raw file system accesses.
2026-01-10 03:10:33 +03:00
Sergei Zimmerman
f672f1a740 libstore: Use makeFSStoreAccessor in derivation builder
With the addition of the file descriptor based source accessor (that does caching)
we now create a fresh instance of the accessor and the last path component is not
followed, so the transformation is safe (actually tests fail without this because
some directories get unlinked and result in ENOENT if the file descriptor gets cached).
2026-01-10 03:10:32 +03:00
Sergei Zimmerman
f20422c53d libutil: Implement unix source accessors that work with file descriptors 2026-01-10 03:10:31 +03:00
Sergei Zimmerman
068e686f0c libutil/unix/file-descriptor: Add a dirFd callback for iterative openFileEnsureBeneathNoSymlinks
This would be useful for caching parent directory file descriptors in the source accessor.
2026-01-10 03:10:30 +03:00
Sergei Zimmerman
3798eb8efd libutil: Add unix::readLinkAt function
This will be used in a following commit.
2026-01-10 03:10:29 +03:00
Sergei Zimmerman
d27e4ed963 libutil: Factor out posixStatToAccessorStat
This will get reused in the UNIX implementations.
2026-01-10 03:10:28 +03:00
Sergei Zimmerman
de7c287c84 libutil/union-source-accessor: Barf on non-existent directories
Previously builtins.readDir would return an empty attribute set
instead of barfing on non-existent paths. This is a regression from
2.32 for impure eval.
2026-01-10 03:10:27 +03:00
Sergei Zimmerman
550a98b34c libstore: Fix mingw build
This also adds a utility for opening a file descriptor from a path in readonly mode.
Previous commit helps a bit with error handling, since now we just throw a NativeSysError.
2026-01-10 03:10:26 +03:00
Sergei Zimmerman
ffb32ae363 libutil: Inline windows-error.hh into error.hh
This way each consumer of NativeSysError doesn't have to
also conditionally include the windows-error.hh, which is very cumbersome.
And we can't include windows-error.hh in error.hh because of a circular import.
2026-01-10 03:10:20 +03:00
53 changed files with 989 additions and 228 deletions

View File

@@ -157,7 +157,7 @@ MixFlakeOptions::MixFlakeOptions()
.category = category,
.labels = {"flake-lock-path"},
.handler = {[&](std::string lockFilePath) {
lockFlags.referenceLockFilePath = {getFSSourceAccessor(), CanonPath(absPath(lockFilePath))};
lockFlags.referenceLockFilePath = {makeFSSourceAccessor(absPath(lockFilePath)), CanonPath::root};
}},
.completer = completePath,
});

View File

@@ -213,26 +213,24 @@ struct MercurialInputScheme : InputScheme
runHg({"status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0"}),
"\0"s);
std::filesystem::path actualPath(absPath(actualUrl));
auto accessor = makeFSSourceAccessor(absPath(actualUrl));
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualPath.string()));
std::string file(p, actualPath.string().size() + 1);
auto cp = CanonPath(p);
auto st = accessor->lstat(cp);
auto st = lstat(p);
if (S_ISDIR(st.st_mode)) {
auto prefix = file + "/";
if (st.type == SourceAccessor::tDirectory) {
auto prefix = cp.rel() + "/";
auto i = files.lower_bound(prefix);
return i != files.end() && hasPrefix(*i, prefix);
}
return files.count(file);
return files.count(cp.rel());
};
auto storePath = store.addToStore(
input.getName(),
{getFSSourceAccessor(), CanonPath(actualPath.string())},
{accessor, CanonPath::root},
ContentAddressMethod::Raw::NixArchive,
HashAlgorithm::SHA256,
{},
@@ -334,7 +332,7 @@ struct MercurialInputScheme : InputScheme
deletePath(tmpDir / ".hg_archival.txt");
auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir.string())});
auto storePath = store.addToStore(name, {makeFSSourceAccessor(tmpDir), CanonPath::root});
Attrs infoAttrs({
{"revCount", (uint64_t) revCount},

View File

@@ -1,5 +1,3 @@
#include "nix/util/archive.hh"
#include "nix/util/posix-source-accessor.hh"
#include "nix/store/store-api.hh"
#include "nix/store/local-fs-store.hh"
#include "nix/store/globals.hh"
@@ -40,13 +38,14 @@ LocalFSStore::LocalFSStore(const Config & config)
{
}
struct LocalStoreAccessor : PosixSourceAccessor
struct LocalStoreAccessor : SourceAccessor
{
ref<SourceAccessor> accessor;
ref<LocalFSStore> store;
bool requireValidPath;
LocalStoreAccessor(ref<LocalFSStore> store, bool requireValidPath)
: PosixSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()})
: accessor(makeFSSourceAccessor(std::filesystem::path{store->config.realStoreDir.get()}))
, store(store)
, requireValidPath(requireValidPath)
{
@@ -67,25 +66,67 @@ struct LocalStoreAccessor : PosixSourceAccessor
return Stat{.type = tDirectory};
requireStoreObject(path);
return PosixSourceAccessor::maybeLstat(path);
return accessor->maybeLstat(path);
}
Stat lstat(const CanonPath & path) override
{
/* Also allow `path` to point to the entire store, which is
needed for resolving symlinks. */
if (path.isRoot())
return Stat{.type = tDirectory};
requireStoreObject(path);
return accessor->lstat(path);
}
DirEntries readDirectory(const CanonPath & path) override
{
requireStoreObject(path);
return PosixSourceAccessor::readDirectory(path);
return accessor->readDirectory(path);
}
std::string readFile(const CanonPath & path) override
{
requireStoreObject(path);
return accessor->readFile(path);
}
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override
{
requireStoreObject(path);
return PosixSourceAccessor::readFile(path, sink, sizeCallback);
return accessor->readFile(path, sink, sizeCallback);
}
std::string readLink(const CanonPath & path) override
{
requireStoreObject(path);
return PosixSourceAccessor::readLink(path);
return accessor->readLink(path);
}
std::string showPath(const CanonPath & path) override
{
return accessor->showPath(path);
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
{
return accessor->getPhysicalPath(path);
}
std::pair<CanonPath, std::optional<std::string>> getFingerprint(const CanonPath & path) override
{
return accessor->getFingerprint(path);
}
std::optional<time_t> getLastModified() override
{
return accessor->getLastModified();
}
bool pathExists(const CanonPath & path) override
{
return accessor->pathExists(path);
}
};
@@ -109,7 +150,7 @@ std::shared_ptr<SourceAccessor> LocalFSStore::getFSAccessor(const StorePath & pa
if (!pathExists(absPath))
return nullptr;
}
return std::make_shared<PosixSourceAccessor>(std::move(absPath));
return makeFSSourceAccessor(std::move(absPath));
}
const std::string LocalFSStore::drvsLogDir = "drvs";

View File

@@ -1059,8 +1059,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
if (info.ca) {
auto & specified = *info.ca;
auto actualHash = ({
auto accessor = getFSAccessor(false);
CanonPath path{info.path.to_string()};
SourcePath sourcePath = requireStoreObjectAccessor(info.path, /*requireValidPath=*/false);
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
auto fim = specified.method.getFileIngestionMethod();
switch (fim) {
@@ -1070,12 +1069,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF
specified.hash.algo,
std::string{info.path.hashPart()},
};
dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim);
dumpPath(sourcePath, caSink, (FileSerialisationMethod) fim);
h = caSink.finish().hash;
break;
}
case FileIngestionMethod::Git:
h = git::dumpHash(specified.hash.algo, {accessor, path}).hash;
h = git::dumpHash(specified.hash.algo, sourcePath).hash;
break;
}
ContentAddress{

View File

@@ -21,7 +21,9 @@ RemoteFSAccessor::RemoteFSAccessor(
std::filesystem::path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{
assert(cacheDir);
return (*cacheDir / hashPart) + "." + ext;
auto res = (*cacheDir / hashPart);
res.concat(concatStrings(".", ext));
return res;
}
ref<SourceAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar)

View File

@@ -1638,12 +1638,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
HashModuloSink caSink{outputHash.hashAlgo, oldHashPart};
auto fim = outputHash.method.getFileIngestionMethod();
dumpPath(
{getFSSourceAccessor(), CanonPath(actualPath.native())}, caSink, (FileSerialisationMethod) fim);
{makeFSSourceAccessor(actualPath), CanonPath::root}, caSink, (FileSerialisationMethod) fim);
return caSink.finish().hash;
}
case FileIngestionMethod::Git: {
return git::dumpHash(outputHash.hashAlgo, {getFSSourceAccessor(), CanonPath(actualPath.native())})
.hash;
return git::dumpHash(outputHash.hashAlgo, {makeFSSourceAccessor(actualPath), CanonPath::root}).hash;
}
}
assert(false);
@@ -1665,7 +1664,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
{
HashResult narHashAndSize = hashPath(
{getFSSourceAccessor(), CanonPath(actualPath.native())},
{makeFSSourceAccessor(actualPath), CanonPath::root},
FileSerialisationMethod::NixArchive,
HashAlgorithm::SHA256);
newInfo0.narHash = narHashAndSize.hash;
@@ -1689,7 +1688,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
std::string{scratchPath->hashPart()}, std::string{requiredFinalPath.hashPart()});
rewriteOutput(outputRewrites);
HashResult narHashAndSize = hashPath(
{getFSSourceAccessor(), CanonPath(actualPath.native())},
{makeFSSourceAccessor(actualPath), CanonPath::root},
FileSerialisationMethod::NixArchive,
HashAlgorithm::SHA256);
ValidPathInfo newInfo0{requiredFinalPath, {store, narHashAndSize.hash}};
@@ -1704,10 +1703,18 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs()
[&](const DerivationOutput::CAFixed & dof) {
auto & wanted = dof.ca.hash;
// Replace the output by a fresh copy of itself to make sure
// that there's no stale file descriptor pointing to it
/* Replace the output by a fresh copy of itself to make sure
that there's no stale file descriptor pointing to it.
IMPORTANT: Copying and deletion must be done in a race-free manner, thus
we are using the source accessor and sink here, since they are implemented
the most robustly. DO NOT USE copyFile here. */
std::filesystem::path tmpOutput = actualPath.native() + ".tmp";
copyFile(actualPath, tmpOutput, true);
auto accessor = makeFSSourceAccessor(actualPath);
auto copySink = RestoreSink(/*startFsync=*/settings.fsyncStorePaths);
copySink.dstPath = tmpOutput;
copyRecursive(*accessor, /*sourcePath=*/CanonPath::root, copySink, /*destPath=*/CanonPath::root);
deletePath(actualPath);
std::filesystem::rename(tmpOutput, actualPath);

View File

@@ -7,7 +7,6 @@
# include <errhandlingapi.h>
# include <fileapi.h>
# include <windows.h>
# include "nix/util/windows-error.hh"
namespace nix {

View File

@@ -119,15 +119,20 @@ TEST_F(FSSourceAccessorTest, works)
EXPECT_THROW(accessor->readFile(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
EXPECT_THROW(accessor->maybeLstat(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), SymlinkNotAllowed);
EXPECT_THROW(accessor->readDirectory(CanonPath("a/dirlink")), NotADirectory);
EXPECT_THROW(accessor->pathExists(CanonPath("a/dirlink/file2")), SymlinkNotAllowed);
}
{
auto accessor = makeFSSourceAccessor(tmpDir / "nonexistent");
EXPECT_FALSE(accessor->maybeLstat(CanonPath::root));
EXPECT_THROW(accessor->readFile(CanonPath::root), SystemError);
}
#ifndef _WIN32
EXPECT_THAT(
[this]() { makeFSSourceAccessor(tmpDir / "nonexistent"); },
::testing::Throws<SysError>(::testing::Field(&SysError::errNo, ENOENT)));
EXPECT_THAT(
[this]() { makeFSSourceAccessor(tmpDir / "nonexistent" / "file"); },
::testing::Throws<SysError>(::testing::Field(&SysError::errNo, ENOENT)));
#endif
EXPECT_THAT(makeFSSourceAccessor(tmpDir / "a" / "dirlink"), HasSymlink(CanonPath::root, "../subdir"));
{
auto accessor = makeFSSourceAccessor(tmpDir, true);

View File

@@ -1,4 +1,5 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "nix/util/file-descriptor.hh"
#include "nix/util/file-system.hh"
@@ -194,4 +195,57 @@ TEST(fchmodatTryNoFollow, fallbackWithoutProc)
}
#endif
/* ----------------------------------------------------------------------------
* readLinkAt
* --------------------------------------------------------------------------*/
TEST(readLinkAt, works)
{
std::filesystem::path tmpDir = nix::createTempDir();
nix::AutoDelete delTmpDir(tmpDir, /*recursive=*/true);
std::string mediumTarget(PATH_MAX / 2, 'x');
std::string longTarget(PATH_MAX - 1, 'y');
{
RestoreSink sink(/*startFsync=*/false);
sink.dstPath = tmpDir;
sink.dirFd = openDirectory(tmpDir);
sink.createSymlink(CanonPath("link"), "target");
sink.createSymlink(CanonPath("relative"), "../relative/path");
sink.createSymlink(CanonPath("absolute"), "/absolute/path");
sink.createSymlink(CanonPath("medium"), mediumTarget);
sink.createSymlink(CanonPath("long"), longTarget);
sink.createDirectory(CanonPath("a"));
sink.createDirectory(CanonPath("a/b"));
sink.createSymlink(CanonPath("a/b/link"), "nested_target");
sink.createRegularFile(CanonPath("regular"), [](CreateRegularFileSink &) {});
sink.createDirectory(CanonPath("dir"));
}
AutoCloseFD dirFd = openDirectory(tmpDir);
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("link")), "target");
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("relative")), "../relative/path");
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("absolute")), "/absolute/path");
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("medium")), mediumTarget);
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("long")), longTarget);
EXPECT_EQ(readLinkAt(dirFd.get(), CanonPath("a/b/link")), "nested_target");
AutoCloseFD subDirFd = openDirectory(tmpDir / "a");
EXPECT_EQ(readLinkAt(subDirFd.get(), CanonPath("b/link")), "nested_target");
EXPECT_THAT(
[&] { readLinkAt(dirFd.get(), CanonPath("regular")); },
Throws<SysError>(::testing::Field(&SysError::errNo, EINVAL)));
EXPECT_THAT(
[&] { readLinkAt(dirFd.get(), CanonPath("dir")); },
Throws<SysError>(::testing::Field(&SysError::errNo, EINVAL)));
EXPECT_THAT(
[&] { readLinkAt(dirFd.get(), CanonPath("nonexistent")); },
Throws<SysError>(::testing::Field(&SysError::errNo, ENOENT)));
}
} // namespace nix

View File

@@ -36,10 +36,10 @@ PathFilter defaultPathFilter = [](const Path &) { return true; };
void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter)
{
auto dumpContents = [&](const CanonPath & path) {
auto dumpContents = [&sink](SourceAccessor & accessor, const CanonPath & path) {
sink << "contents";
std::optional<uint64_t> size;
readFile(path, sink, [&](uint64_t _size) {
accessor.readFile(path, sink, [&](uint64_t _size) {
size = _size;
sink << _size;
});
@@ -49,10 +49,10 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
sink << narVersionMagic1;
[&, &this_(*this)](this const auto & dump, const CanonPath & path) -> void {
[&sink, &filter, &dumpContents](this const auto & dump, SourceAccessor & accessor, const CanonPath & path) -> void {
checkInterrupt();
auto st = this_.lstat(path);
auto st = accessor.lstat(path);
sink << "(";
@@ -60,7 +60,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
sink << "type" << "regular";
if (st.isExecutable)
sink << "executable" << "";
dumpContents(path);
dumpContents(accessor, path);
}
else if (st.type == tDirectory) {
@@ -69,7 +69,7 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
/* If we're on a case-insensitive system like macOS, undo
the case hack applied by restorePath(). */
StringMap unhacked;
for (auto & i : this_.readDirectory(path))
for (auto & i : accessor.readDirectory(path))
if (archiveSettings.useCaseHack) {
std::string name(i.first);
size_t pos = i.first.find(caseHackSuffix);
@@ -83,34 +83,37 @@ void SourceAccessor::dumpPath(const CanonPath & path, Sink & sink, PathFilter &
} else
unhacked.emplace(i.first, i.first);
for (auto & i : unhacked)
if (filter((path / i.first).abs())) {
sink << "entry" << "(" << "name" << i.first << "node";
dump(path / i.second);
sink << ")";
}
accessor.readDirectory(path, [&](SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath) {
for (auto & i : unhacked)
if (filter((path / i.first).abs())) {
sink << "entry" << "(" << "name" << i.first << "node";
dump(subdirAccessor, subdirRelPath / i.second);
sink << ")";
}
});
}
else if (st.type == tSymlink)
sink << "type" << "symlink" << "target" << this_.readLink(path);
sink << "type" << "symlink" << "target" << accessor.readLink(path);
else
throw Error("file '%s' has an unsupported type", path);
sink << ")";
}(path);
}(*this, path);
}
time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
{
auto path2 = PosixSourceAccessor::createAtRoot(path, /*trackLastModified=*/true);
SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/true);
path2.dumpPath(sink, filter);
return path2.accessor->getLastModified().value();
}
void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
{
dumpPathAndGetMtime(path, sink, filter);
SourcePath path2 = makeFSSourceAccessor(absPath(path), /*trackLastModified=*/false);
path2.dumpPath(sink, filter);
}
void dumpString(std::string_view s, Sink & sink)

View File

@@ -6,7 +6,6 @@
#ifdef _WIN32
# include <winnt.h>
# include <fileapi.h>
# include "nix/util/windows-error.hh"
#endif
namespace nix {

View File

@@ -7,7 +7,6 @@
#ifdef _WIN32
# include <fileapi.h>
# include "nix/util/file-path.hh"
# include "nix/util/windows-error.hh"
#endif
#include "util-config-private.hh"
@@ -34,10 +33,12 @@ void copyRecursive(SourceAccessor & accessor, const CanonPath & from, FileSystem
}
case SourceAccessor::tDirectory: {
sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPath) {
for (auto & [name, _] : accessor.readDirectory(from)) {
copyRecursive(accessor, from / name, dirSink, relDirPath / name);
}
sink.createDirectory(to, [&](FileSystemObjectSink & dirSink, const CanonPath & relDirPathTo) {
accessor.readDirectory(from, [&](SourceAccessor & subdirAccessor, const CanonPath & relDirPathFrom) {
for (auto & [name, _] : accessor.readDirectory(from)) {
copyRecursive(subdirAccessor, relDirPathFrom / name, dirSink, relDirPathTo / name);
}
});
});
break;
}

View File

@@ -27,6 +27,9 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef _WIN32
# include <errhandlingapi.h>
#endif
namespace nix {
@@ -288,25 +291,6 @@ public:
}
};
#ifdef _WIN32
namespace windows {
class WinError;
}
#endif
/**
* Convenience alias for when we use a `errno`-based error handling
* function on Unix, and `GetLastError()`-based error handling on on
* Windows.
*/
using NativeSysError =
#ifdef _WIN32
windows::WinError
#else
SysError
#endif
;
/**
* Throw an exception for the purpose of checking that exception
* handling works; see 'initLibUtil()'.
@@ -326,4 +310,67 @@ void panic(std::string_view msg);
*/
[[gnu::noinline, gnu::cold, noreturn]] void unreachable(std::source_location loc = std::source_location::current());
#ifdef _WIN32
namespace windows {
/**
* Windows Error type.
*
* Unless you need to catch a specific error number, don't catch this in
* portable code. Catch `SystemError` instead.
*/
class WinError : public SystemError
{
public:
DWORD lastError;
/**
* Construct using the explicitly-provided error number.
* `FormatMessageA` will be used to try to add additional
* information to the message.
*/
template<typename... Args>
WinError(DWORD lastError, const Args &... args)
: SystemError("")
, lastError(lastError)
{
auto hf = HintFmt(args...);
err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), renderError(lastError));
}
/**
* Construct using `GetLastError()` and the ambient "last error".
*
* Be sure to not perform another last-error-modifying operation
* before calling this constructor!
*/
template<typename... Args>
WinError(const Args &... args)
: WinError(GetLastError(), args...)
{
}
private:
std::string renderError(DWORD lastError);
};
} // namespace windows
#endif
/**
* Convenience alias for when we use a `errno`-based error handling
* function on Unix, and `GetLastError()`-based error handling on on
* Windows.
*/
using NativeSysError =
#ifdef _WIN32
windows::WinError
#else
SysError
#endif
;
} // namespace nix

View File

@@ -250,12 +250,18 @@ namespace unix {
*
* @param flags O_* flags
* @param mode Mode for O_{CREAT,TMPFILE}
* @param dirFdCallback Callback invoked that gets the ownership of an intermediate directory fd.
*
* @pre path.isRoot() is false
*
* @throws SymlinkNotAllowed if any path components
*/
Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode = 0);
Descriptor openFileEnsureBeneathNoSymlinks(
Descriptor dirFd,
const CanonPath & path,
int flags,
mode_t mode = 0,
std::function<void(AutoCloseFD dirFd, CanonPath relPath)> dirFdCallback = nullptr);
/**
* Try to change the mode of file named by \ref path relative to the parent directory denoted by \ref dirFd.
@@ -268,6 +274,11 @@ Descriptor openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & p
*/
void fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t mode);
/*
* Read a symlink relative to a directory file descriptor.
*/
std::string readLinkAt(Descriptor dirFd, const CanonPath & path);
} // namespace unix
#endif

View File

@@ -164,6 +164,13 @@ std::filesystem::path readLink(const std::filesystem::path & path);
*/
Descriptor openDirectory(const std::filesystem::path & path);
/**
* Open a `Descriptor` with read-only access to the given file.
*
* @note For directories use @ref openDirectory.
*/
Descriptor openFileReadonly(const std::filesystem::path & path);
/**
* Read the contents of a file into a string.
*/

View File

@@ -10,7 +10,6 @@
# include <poll.h>
#else
# include <ioapiset.h>
# include "nix/util/windows-error.hh"
#endif
namespace nix {

View File

@@ -4,6 +4,7 @@
#include "nix/util/memory-source-accessor.hh"
#include <functional>
#include <filesystem>
#include <nlohmann/json_fwd.hpp>
@@ -30,7 +31,7 @@ using GetNarBytes = std::function<std::string(uint64_t, uint64_t)>;
/**
* The canonical GetNarBytes function for a seekable Source.
*/
GetNarBytes seekableGetNarBytes(const Path & path);
GetNarBytes seekableGetNarBytes(const std::filesystem::path & path);
GetNarBytes seekableGetNarBytes(Descriptor fd);

View File

@@ -43,36 +43,6 @@ public:
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override;
/**
* Create a `PosixSourceAccessor` and `SourcePath` corresponding to
* some native path.
*
* @param Whether the accessor should return a non-null getLastModified.
* When true the accessor must be used only by a single thread.
*
* The `PosixSourceAccessor` is rooted as far up the tree as
* possible, (e.g. on Windows it could scoped to a drive like
* `C:\`). This allows more `..` parent accessing to work.
*
* @note When `path` is trusted user input, canonicalize it using
* `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc,
* as appropriate for the use case. At least weak canonicalization is
* required for the `SourcePath` to do anything useful at the location it
* points to.
*
* @note A canonicalizing behavior is not built in `createAtRoot` so that
* callers do not accidentally introduce symlink-related security vulnerabilities.
* Furthermore, `createAtRoot` does not know whether the file pointed to by
* `path` should be resolved if it is itself a symlink. In other words,
* `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers.
*
* See
* [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path)
* and
* [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path).
*/
static SourcePath createAtRoot(const std::filesystem::path & path, bool trackLastModified = false);
std::optional<std::time_t> getLastModified() override
{
return trackLastModified ? std::optional{mtime} : std::nullopt;

View File

@@ -138,6 +138,22 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
*/
virtual DirEntries readDirectory(const CanonPath & path) = 0;
/**
* Variation of readDirectory that receives a SourceAccessor possibly scoped to \ref dirPath.
* Primary meant for recursive traversal functions that would benefit from *at-style syscalls
* relative to a particular directory.
*
* @note Like `readFile`, this method should *not* follow symlinks.
* @param callback Caller-provided function invoked with a maximally deeply scoped SourceAccessor and the path that
* would have to be prepended to each path relative to dirPath to access a particular file with it.
*/
virtual void readDirectory(
const CanonPath & dirPath,
std::function<void(SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath)> callback)
{
callback(*this, dirPath);
}
virtual std::string readLink(const CanonPath & path) = 0;
virtual void dumpPath(const CanonPath & path, Sink & sink, PathFilter & filter = defaultPathFilter);
@@ -220,6 +236,11 @@ struct SourceAccessor : std::enable_shared_from_this<SourceAccessor>
*/
ref<SourceAccessor> makeEmptySourceAccessor();
/**
* Helper function that's shared between PosixSourceAccessor and UNIX accessors.
*/
SourceAccessor::Stat posixStatToAccessorStat(const struct ::stat & st);
/**
* Exception thrown when accessing a filtered path (see
* `FilteringSourceAccessor`).
@@ -254,6 +275,18 @@ ref<SourceAccessor> getFSSourceAccessor();
* that it is not possible to escape `root` by appending `..` path
* elements, and that absolute symlinks are resolved relative to
* `root`.
*
* @param root Path to the root of the accessor. Must exist. Symlinks in non-last
* components are followed. If \ref root names a symlink an accessor for that symlink
* is returned.
*
* @note Once created the accessor cannot be used if the root of the accessor is modified.
* For example, if \ref root names a directory it will be possible to call readDirectory(CanonPath:root).
* Similarly with symlinks and regular files.
*
* @todo Use file HANDLEs on windows.
*
* @throws SysError if path doesn't exist.
*/
ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified = false);

View File

@@ -1,6 +1,7 @@
#include "nix/util/nar-accessor.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/archive.hh"
#include "nix/util/error.hh"
#include <map>
#include <stack>
@@ -263,17 +264,11 @@ ref<SourceAccessor> makeLazyNarAccessor(Source & source, GetNarBytes getNarBytes
return make_ref<NarAccessor>(source, getNarBytes);
}
GetNarBytes seekableGetNarBytes(const Path & path)
GetNarBytes seekableGetNarBytes(const std::filesystem::path & path)
{
AutoCloseFD fd = toDescriptor(open(
path.c_str(),
O_RDONLY
#ifdef O_CLOEXEC
| O_CLOEXEC
#endif
));
AutoCloseFD fd = openFileReadonly(path);
if (!fd)
throw SysError("opening NAR cache file '%s'", path);
throw NativeSysError("opening NAR cache file '%s'", path);
return [inner = seekableGetNarBytes(fd.get()), fd = make_ref<AutoCloseFD>(std::move(fd))](
uint64_t offset, uint64_t length) { return inner(offset, length); };

View File

@@ -1,7 +1,5 @@
#include "nix/util/posix-source-accessor.hh"
#include "nix/util/source-path.hh"
#include "nix/util/signals.hh"
#include "nix/util/sync.hh"
#include <boost/unordered/concurrent_flat_map.hpp>
@@ -20,15 +18,6 @@ PosixSourceAccessor::PosixSourceAccessor()
{
}
SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path, bool trackLastModified)
{
std::filesystem::path path2 = absPath(path);
return {
make_ref<PosixSourceAccessor>(path2.root_path(), trackLastModified),
CanonPath{path2.relative_path().string()},
};
}
std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path)
{
return root.empty() ? (std::filesystem::path{path.abs()})
@@ -121,22 +110,7 @@ std::optional<SourceAccessor::Stat> PosixSourceAccessor::maybeLstat(const CanonP
if (trackLastModified)
mtime = std::max(mtime, st->st_mtime);
return Stat{
.type = S_ISREG(st->st_mode) ? tRegular
: S_ISDIR(st->st_mode) ? tDirectory
: S_ISLNK(st->st_mode) ? tSymlink
: S_ISCHR(st->st_mode) ? tChar
: S_ISBLK(st->st_mode) ? tBlock
:
#ifdef S_ISSOCK
S_ISSOCK(st->st_mode) ? tSocket
:
#endif
S_ISFIFO(st->st_mode) ? tFifo
: tUnknown,
.fileSize = S_ISREG(st->st_mode) ? std::optional<uint64_t>(st->st_size) : std::nullopt,
.isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR,
};
return posixStatToAccessorStat(*st);
}
SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path)
@@ -213,6 +187,7 @@ void PosixSourceAccessor::assertNoSymlinks(CanonPath path)
}
}
#ifdef _WIN32
ref<SourceAccessor> getFSSourceAccessor()
{
static auto rootFS = make_ref<PosixSourceAccessor>();
@@ -223,4 +198,6 @@ ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackL
{
return make_ref<PosixSourceAccessor>(std::move(root), trackLastModified);
}
#endif
} // namespace nix

View File

@@ -13,7 +13,6 @@
#ifdef _WIN32
# include <fileapi.h>
# include "nix/util/windows-error.hh"
#else
# include <poll.h>
#endif

View File

@@ -76,6 +76,27 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path)
throw FileNotFound("path '%s' does not exist", showPath(path));
}
SourceAccessor::Stat posixStatToAccessorStat(const struct ::stat & st)
{
using enum SourceAccessor::Type;
return SourceAccessor::Stat{
.type = S_ISREG(st.st_mode) ? tRegular
: S_ISDIR(st.st_mode) ? tDirectory
: S_ISLNK(st.st_mode) ? tSymlink
: S_ISCHR(st.st_mode) ? tChar
: S_ISBLK(st.st_mode) ? tBlock
:
#ifdef S_ISSOCK
S_ISSOCK(st.st_mode) ? tSocket
:
#endif
S_ISFIFO(st.st_mode) ? tFifo
: tUnknown,
.fileSize = S_ISREG(st.st_mode) ? std::optional<uint64_t>(st.st_size) : std::nullopt,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR,
};
}
void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix)
{
this->displayPrefix = std::move(displayPrefix);

View File

@@ -35,14 +35,18 @@ struct UnionSourceAccessor : SourceAccessor
DirEntries readDirectory(const CanonPath & path) override
{
DirEntries result;
bool exists = false;
for (auto & accessor : accessors) {
auto st = accessor->maybeLstat(path);
if (!st)
continue;
exists = true;
for (auto & entry : accessor->readDirectory(path))
// Don't override entries from previous accessors.
result.insert(entry);
}
if (!exists)
throw FileNotFound("path '%s' does not exist", showPath(path));
return result;
}

View File

@@ -340,20 +340,27 @@ void unix::fchmodatTryNoFollow(Descriptor dirFd, const CanonPath & path, mode_t
throw SysError("fchmodat '%s' relative to parent directory", path.rel());
}
static Descriptor
openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode)
static Descriptor openFileEnsureBeneathNoSymlinksIterative(
Descriptor dirFd,
const CanonPath & path,
int flags,
mode_t mode,
std::function<void(AutoCloseFD dirFd, CanonPath relPath)> dirFdCallback)
{
AutoCloseFD parentFd;
auto nrComponents = std::ranges::distance(path);
assert(nrComponents >= 1);
auto components = std::views::take(path, nrComponents - 1); /* Everything but last component */
auto getParentFd = [&]() { return parentFd ? parentFd.get() : dirFd; };
auto currentRelPath = CanonPath::root;
/* This rather convoluted loop is necessary to avoid TOCTOU when validating that
no inner path component is a symlink. */
for (auto it = components.begin(); it != components.end(); ++it) {
auto component = std::string(*it); /* Copy into a string to make NUL terminated. */
assert(component != ".." && !component.starts_with('/')); /* In case invariant is broken somehow.. */
auto prevRelPath = currentRelPath;
currentRelPath = currentRelPath / *it;
AutoCloseFD parentFd2 = ::openat(
getParentFd(), /* First iteration uses dirFd. */
@@ -386,6 +393,9 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat
return INVALID_DESCRIPTOR;
}
if (dirFdCallback && parentFd)
dirFdCallback(std::move(parentFd), std::move(prevRelPath));
parentFd = std::move(parentFd2);
}
@@ -395,7 +405,12 @@ openFileEnsureBeneathNoSymlinksIterative(Descriptor dirFd, const CanonPath & pat
return res;
}
Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPath & path, int flags, mode_t mode)
Descriptor unix::openFileEnsureBeneathNoSymlinks(
Descriptor dirFd,
const CanonPath & path,
int flags,
mode_t mode,
std::function<void(AutoCloseFD dirFd, CanonPath relPath)> dirFdCallback)
{
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
assert(!path.isRoot());
@@ -408,7 +423,23 @@ Descriptor unix::openFileEnsureBeneathNoSymlinks(Descriptor dirFd, const CanonPa
return *maybeFd;
}
#endif
return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode);
return openFileEnsureBeneathNoSymlinksIterative(dirFd, path, flags, mode, dirFdCallback);
}
std::string unix::readLinkAt(Descriptor dirFd, const CanonPath & path)
{
assert(!path.isRoot());
assert(!path.rel().starts_with('/')); /* Just in case the invariant is somehow broken. */
std::vector<char> buf;
for (ssize_t bufSize = PATH_MAX / 4; true; bufSize += bufSize / 2) {
checkInterrupt();
buf.resize(bufSize);
ssize_t rlSize = ::readlinkat(dirFd, path.rel_c_str(), buf.data(), bufSize);
if (rlSize == -1)
throw SysError("reading symbolic link '%1%'", path);
else if (rlSize < bufSize)
return {buf.data(), static_cast<std::size_t>(rlSize)};
}
}
} // namespace nix

View File

@@ -27,6 +27,11 @@ Descriptor openDirectory(const std::filesystem::path & path)
return open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC);
}
Descriptor openFileReadonly(const std::filesystem::path & path)
{
return open(path.c_str(), O_RDONLY | O_CLOEXEC);
}
std::filesystem::path defaultTempDir()
{
return getEnvNonEmpty("TMPDIR").value_or("/tmp");

View File

@@ -0,0 +1,165 @@
#pragma once
///@file
#include <cassert>
#include <mutex>
#include "nix/util/lru-cache.hh"
#include "nix/util/signals.hh"
#include "nix/util/source-accessor.hh"
#include "nix/util/file-descriptor.hh"
#include "nix/util/sync.hh"
namespace nix {
namespace unix {
/* The accessors for file/directory access are different, because we want them
all to work with file descriptors. Technically that could be done on Linux using
O_PATH descriptors, but that wouldn't work on Darwin. */
class UnixSourceAccessorBase : public SourceAccessor
{
protected:
bool trackLastModified;
/**
* The most recent mtime seen by fstat(). This is a hack to
* support dumpPathAndGetMtime(). Should remove this eventually.
*/
std::time_t mtime = 0;
UnixSourceAccessorBase(bool trackLastModified)
: trackLastModified(trackLastModified)
{
}
void updateMtime(std::time_t newMtime)
{
/* The contract is that trackLastModified implies that the caller uses the accessor
from a single thread. Thus this is not a CAS loop. */
if (trackLastModified)
mtime = std::max(mtime, newMtime);
}
public:
std::optional<std::time_t> getLastModified() override
{
return trackLastModified ? std::optional{mtime} : std::nullopt;
}
};
class UnixFileSourceAccessor : public UnixSourceAccessorBase
{
AutoCloseFD fd;
CanonPath rootPath;
mutable std::once_flag statFlag;
mutable struct ::stat cachedStat;
public:
UnixFileSourceAccessor(AutoCloseFD fd_, CanonPath rootPath_, bool trackLastModified, struct ::stat * st = nullptr)
: UnixSourceAccessorBase(trackLastModified)
, fd(std::move(fd_))
, rootPath(std::move(rootPath_))
{
displayPrefix = rootPath.abs();
if (st) {
std::call_once(statFlag, [this, st] {
cachedStat = *st;
updateMtime(cachedStat.st_mtime);
});
}
}
std::string showPath(const CanonPath & path) override
{
if (path.isRoot())
return displayPrefix; /* No trailing slash, we know it's not a directory. */
return SourceAccessor::showPath(path);
}
DirEntries readDirectory(const CanonPath & path) override
{
if (!path.isRoot())
throw FileNotFound("path '%s' does not exist", showPath(path));
throw NotADirectory("path '%s' is not a directory", showPath(path));
}
std::string readLink(const CanonPath & path) override
{
if (!path.isRoot())
throw FileNotFound("path '%s' does not exist", showPath(path));
throw NotASymlink("path '%s' is not a symlink", showPath(path));
}
bool pathExists(const CanonPath & path) override
{
return path.isRoot(); /* We know that we are accessing a regular file and not a directory. */
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
{
if (path.isRoot())
return std::filesystem::path(rootPath.abs());
/* Slightly different than what PosixSourceAccessor used to do, but we know that this is not a directory. */
return std::nullopt;
}
std::optional<Stat> maybeLstat(const CanonPath & path) override;
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
};
class UnixDirectorySourceAccessor : public UnixSourceAccessorBase
{
AutoCloseFD fd;
CanonPath rootPath;
std::unique_ptr<Sync<LRUCache<CanonPath, ref<AutoCloseFD>>>> dirFdCache;
/**
* Get the parent directory of path. The second pair element might be an owning file descriptor
* if path.parent().isRoot() is false.
*/
std::pair<Descriptor, std::shared_ptr<AutoCloseFD>> openParent(const CanonPath & path);
std::function<void(AutoCloseFD, CanonPath)> makeDirFdCallback();
AutoCloseFD openSubdirectory(const CanonPath & path);
public:
UnixDirectorySourceAccessor(
AutoCloseFD fd_, CanonPath rootPath_, bool trackLastModified, unsigned dirFdCacheSize = 128)
: UnixSourceAccessorBase(trackLastModified)
, fd(std::move(fd_))
, rootPath(std::move(rootPath_))
{
if (rootPath.isRoot())
displayPrefix.clear(); /* To avoid the double slash. */
else
displayPrefix = rootPath.abs();
if (dirFdCacheSize)
dirFdCache = std::make_unique<Sync<LRUCache<CanonPath, ref<AutoCloseFD>>>>(dirFdCacheSize);
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
{
if (path.isRoot())
return std::filesystem::path(rootPath.abs());
return std::filesystem::path(rootPath.abs()) / path.rel(); /* RHS *must* be a relative path. */
}
std::optional<Stat> maybeLstat(const CanonPath & path) override;
void readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback) override;
DirEntries readDirectory(const CanonPath & path) override;
void readDirectory(
const CanonPath & dirPath,
std::function<void(SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath)> callback) override;
std::string readLink(const CanonPath & path) override;
};
} // namespace unix
} // namespace nix

View File

@@ -58,6 +58,7 @@ sources += files(
'os-string.cc',
'processes.cc',
'signals.cc',
'unix-source-accessor.cc',
'users.cc',
)

View File

@@ -0,0 +1,358 @@
#include "nix/util/memory-source-accessor.hh"
#include "nix/util/source-accessor.hh"
#include "nix/util/unix-source-accessor.hh"
namespace nix {
using namespace nix::unix;
std::optional<SourceAccessor::Stat> UnixFileSourceAccessor::maybeLstat(const CanonPath & path)
{
if (!path.isRoot())
/* This is not a directory. Nothing can be beneath it. */
return std::nullopt;
std::call_once(statFlag, [this] {
if (::fstat(fd.get(), &cachedStat) == -1)
throw SysError("statting file '%s'", displayPrefix);
updateMtime(cachedStat.st_mtime);
});
return posixStatToAccessorStat(cachedStat);
}
void UnixFileSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
{
if (!path.isRoot())
throw FileNotFound("path '%s' does not exist", showPath(path));
struct ::stat st;
/* Fresh fstat. TODO: Maybe reuse the cached stat? There are some nuances when it comes
to non-regular file handling (e.g. /dev/stdin) that is also system dependent.
See https://github.com/NixOS/nix/issues/9330. We should probably ban non-regular files
completely. */
if (::fstat(fd.get(), &st) == -1)
throw SysError("getting status of '%s'", showPath(path));
off_t left = st.st_size;
off_t offset = 0;
/* Currently trusts st_size to be correct, errors out if EOF is reached before reading st_size bytes:
https://github.com/NixOS/nix/issues/10667. */
sizeCallback(left);
/* TODO: Optimise for the case when Sink is an FdSink and call sendfile. Can
also use copy_file_range to leverage reflinking if the destination is a
regular file and not a socket. */
std::array<unsigned char, 64 * 1024> buf;
while (left) {
checkInterrupt();
/* N.B. Using pread for thread-safety. File pointer must not be modified. */
ssize_t rd = ::pread(fd.get(), buf.data(), std::min<std::size_t>(left, buf.size()), offset);
if (rd == -1) {
if (errno != EINTR)
throw SysError("reading from file '%s'", showPath(path));
} else if (rd == 0)
throw SysError("unexpected end-of-file reading '%s'", showPath(path));
else {
assert(rd <= left);
sink({reinterpret_cast<char *>(buf.data()), static_cast<std::size_t>(rd)});
left -= rd;
offset += rd;
}
}
}
std::function<void(AutoCloseFD, CanonPath)> UnixDirectorySourceAccessor::makeDirFdCallback()
{
if (!dirFdCache)
return nullptr;
return [this](AutoCloseFD fd, CanonPath key) {
auto cache(dirFdCache->lock());
assert(fd);
cache->upsert(key, make_ref<AutoCloseFD>(std::move(fd)));
};
}
std::pair<Descriptor, std::shared_ptr<AutoCloseFD>> UnixDirectorySourceAccessor::openParent(const CanonPath & path)
{
assert(!path.isRoot());
auto parent = path.parent().value();
if (parent.isRoot())
return {fd.get(), nullptr};
if (dirFdCache) {
if (auto cachedFd = dirFdCache->lock()->get(parent)) {
assert((*cachedFd)->get());
return {(*cachedFd)->get(), *cachedFd};
}
}
AutoCloseFD parentFdOwning = openFileEnsureBeneathNoSymlinks(
fd.get(), parent, O_DIRECTORY | O_RDONLY | O_NOFOLLOW | O_CLOEXEC, 0, makeDirFdCallback());
if (!parentFdOwning && (errno == ELOOP || errno == ENOTDIR))
throw SymlinkNotAllowed(parent);
return {parentFdOwning.get(), make_ref<AutoCloseFD>(std::move(parentFdOwning))};
}
std::optional<SourceAccessor::Stat> UnixDirectorySourceAccessor::maybeLstat(const CanonPath & path)
try {
struct ::stat st;
if (path.isRoot()) {
/* This error is unexpected. Would only happen if the directory fd is messed up. */
if (::fstat(fd.get(), &st) == -1)
throw SysError("getting status of '%s'", showPath(path));
} else {
auto [parentFd, parentFdOwning] = openParent(path);
if (parentFd == INVALID_DESCRIPTOR)
return std::nullopt;
if (::fstatat(parentFd, std::string(path.baseName().value()).c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1)
return std::nullopt;
if (dirFdCache && parentFdOwning) {
assert(*parentFdOwning);
dirFdCache->lock()->upsert(path.parent().value(), ref<AutoCloseFD>(parentFdOwning));
}
}
updateMtime(st.st_mtime);
return posixStatToAccessorStat(st);
} catch (SymlinkNotAllowed & e) {
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
}
void UnixDirectorySourceAccessor::readFile(
const CanonPath & path, Sink & sink, std::function<void(uint64_t)> sizeCallback)
try {
if (path.isRoot())
throw NotARegularFile("'%s' is not a regular file", showPath(path));
AutoCloseFD fileFd = openFileEnsureBeneathNoSymlinks(fd.get(), path, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
if (!fileFd) {
if (errno == ELOOP) /* The last component is a symlink. */
throw NotARegularFile("'%s' is a symlink, not a regular file", showPath(path));
if (errno == ENOENT || errno == ENOTDIR) /* Intermediate component might not exist. */
throw FileNotFound("file '%s' does not exist", showPath(path));
throw SysError("opening '%s'", showPath(path));
}
UnixFileSourceAccessor fileAccessor(std::move(fileFd), rootPath / path, trackLastModified);
fileAccessor.readFile(CanonPath::root, sink, sizeCallback);
if (auto fileMtime = fileAccessor.getLastModified())
mtime = std::max(mtime, *fileMtime);
} catch (SymlinkNotAllowed & e) {
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
}
AutoCloseFD UnixDirectorySourceAccessor::openSubdirectory(const CanonPath & path)
try {
AutoCloseFD dirFdOwning;
if (path.isRoot()) {
/* Get a fresh file descriptor for thread-safety. */
dirFdOwning = ::openat(fd.get(), ".", O_DIRECTORY | O_RDONLY | O_CLOEXEC);
if (!dirFdOwning)
throw SysError("opening directory '%s'", showPath(path));
} else {
dirFdOwning = openFileEnsureBeneathNoSymlinks(
fd.get(), path, O_DIRECTORY | O_RDONLY | O_NOFOLLOW | O_CLOEXEC, 0, makeDirFdCallback());
if (!dirFdOwning) {
if (errno == ENOTDIR)
throw NotADirectory("'%s' is not a directory", showPath(path));
throw SysError("opening directory '%s'", showPath(path));
}
}
return dirFdOwning;
} catch (SymlinkNotAllowed & e) {
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
}
SourceAccessor::DirEntries UnixDirectorySourceAccessor::readDirectory(const CanonPath & path)
try {
AutoCloseFD dirFdOwning = openSubdirectory(path);
AutoCloseDir dir(::fdopendir(dirFdOwning.get()));
if (!dir)
throw SysError("opening directory '%s'", showPath(path));
dirFdOwning.release();
DirEntries entries;
const ::dirent * dirent = nullptr;
while (errno = 0, dirent = ::readdir(dir.get())) {
checkInterrupt();
std::string_view name(dirent->d_name);
if (name == "." || name == "..")
continue;
std::optional<Type> type;
switch (dirent->d_type) {
case DT_REG:
type = tRegular;
break;
case DT_DIR:
type = tDirectory;
break;
case DT_LNK:
type = tSymlink;
break;
case DT_CHR:
type = tChar;
break;
case DT_BLK:
type = tBlock;
break;
case DT_FIFO:
type = tFifo;
break;
case DT_SOCK:
type = tSocket;
break;
default:
type = std::nullopt;
break;
}
entries.emplace(name, type);
}
if (errno)
throw SysError("reading directory %1%", path);
return entries;
} catch (SymlinkNotAllowed & e) {
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
}
void UnixDirectorySourceAccessor::readDirectory(
const CanonPath & dirPath,
std::function<void(SourceAccessor & subdirAccessor, const CanonPath & subdirRelPath)> callback)
{
auto fd = openSubdirectory(dirPath);
UnixDirectorySourceAccessor accessor{std::move(fd), rootPath / dirPath, trackLastModified, /*dirFdCacheSize=*/0};
callback(accessor, CanonPath::root);
}
std::string UnixDirectorySourceAccessor::readLink(const CanonPath & path)
try {
if (path.isRoot())
throw NotASymlink("file '%s' is not a symlink", showPath(path));
auto [parentFd, parentFdOwning] = openParent(path);
if (parentFd == INVALID_DESCRIPTOR)
throw FileNotFound("file '%s' does not exist", showPath(path));
if (dirFdCache && parentFdOwning) {
assert(*parentFdOwning);
dirFdCache->lock()->upsert(path.parent().value(), ref<AutoCloseFD>(std::move(parentFdOwning)));
}
try {
return readLinkAt(parentFd, CanonPath(path.baseName().value()));
} catch (SysError & e) {
if (e.errNo == EINVAL)
throw NotASymlink("file '%s' is not a symlink", showPath(path));
throw;
}
} catch (SymlinkNotAllowed & e) {
throw SymlinkNotAllowed(e.path, "path '%s' is a symlink", showPath(e.path));
}
namespace {
class SymlinkSourceAccessor : public MemorySourceAccessor
{
bool trackLastModified;
std::time_t mtime;
CanonPath rootPath;
public:
SymlinkSourceAccessor(std::string target, CanonPath rootPath_, bool trackLastModified, std::time_t mtime)
: trackLastModified(trackLastModified)
, mtime(mtime)
, rootPath(std::move(rootPath_))
{
MemorySink sink{*this};
sink.createSymlink(CanonPath::root, target);
displayPrefix = rootPath.abs();
}
std::optional<std::time_t> getLastModified() override
{
return trackLastModified ? std::optional{mtime} : std::nullopt;
}
std::optional<std::filesystem::path> getPhysicalPath(const CanonPath & path) override
{
auto fsRootPath = std::filesystem::path(rootPath.abs());
if (path.isRoot())
return fsRootPath;
return fsRootPath / path.rel(); /* RHS must be a relative path. */
}
std::string showPath(const CanonPath & path) override
{
/* When rendering the file itself omit the trailing slash. */
return path.isRoot() ? displayPrefix : SourceAccessor::showPath(path);
}
};
} // namespace
ref<SourceAccessor> getFSSourceAccessor()
{
static auto rootFS =
make_ref<UnixDirectorySourceAccessor>(openDirectory("/"), CanonPath("/"), /*trackLastModified=*/false);
return rootFS;
}
ref<SourceAccessor> makeFSSourceAccessor(std::filesystem::path root, bool trackLastModified)
{
using namespace unix;
if (root.empty())
return getFSSourceAccessor();
assert(root.is_absolute());
auto rootPath = CanonPath(root.native());
assert(rootPath.abs().starts_with("/")); /* In case the invariant is broken somehow. */
/* Any symlinks get resolved eagerly here. Unlike with SourceAccessor semantics that requires that
all links get resolved manually, the root can be resolved eagerly. */
AutoCloseFD fd(::open(rootPath.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW));
if (!fd) {
if (errno == ELOOP) /* Opening a symlink, can read it straight into memory source accessor. */ {
auto parent = rootPath.parent().value(); /* Always present, isRoot is handled above. */
auto name = std::string(rootPath.baseName().value());
AutoCloseFD parentFd(::open(parent.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC));
if (!parentFd)
throw SysError("opening '%s'", parent.abs());
struct ::stat st;
if (::fstatat(parentFd.get(), name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1)
throw SysError("statting '%s' relative to parent directory '%s'", name, parent.abs());
auto target = readLinkAt(parentFd.get(), CanonPath(name));
return make_ref<SymlinkSourceAccessor>(
std::move(target), std::move(rootPath), trackLastModified, st.st_mtime);
}
throw SysError("opening '%s'", rootPath.abs());
}
struct ::stat st;
if (::fstat(fd.get(), &st) == -1)
throw SysError("statting '%s'", rootPath.abs());
if (S_ISDIR(st.st_mode))
return make_ref<UnixDirectorySourceAccessor>(std::move(fd), std::move(rootPath), trackLastModified, /*dirFdCacheSize=*/0);
/* TODO: Ban non-regular files that cannot file represented by the FSO
semantics. See comment in UnixFileSourceAccessor::readFile. */
return make_ref<UnixFileSourceAccessor>(std::move(fd), std::move(rootPath), trackLastModified, &st);
}
} // namespace nix

View File

@@ -1,5 +1,5 @@
#include "nix/util/current-process.hh"
#include "nix/util/windows-error.hh"
#include "nix/util/error.hh"
#include <cmath>
#ifdef _WIN32

View File

@@ -2,7 +2,6 @@
#include "nix/util/signals.hh"
#include "nix/util/finally.hh"
#include "nix/util/serialise.hh"
#include "nix/util/windows-error.hh"
#include "nix/util/file-path.hh"
#include <fileapi.h>

View File

@@ -1,5 +1,4 @@
#include "nix/util/file-system.hh"
#include "nix/util/windows-error.hh"
#include "nix/util/logging.hh"
namespace nix {
@@ -24,10 +23,22 @@ Descriptor openDirectory(const std::filesystem::path & path)
path.c_str(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
NULL,
/*lpSecurityAttributes=*/nullptr,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS,
NULL);
/*hTemplateFile=*/nullptr);
}
Descriptor openFileReadonly(const std::filesystem::path & path)
{
return CreateFileW(
path.c_str(),
GENERIC_READ,
FILE_SHARE_READ | FILE_SHARE_DELETE,
/*lpSecurityAttributes=*/nullptr,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
/*hTemplateFile=*/nullptr);
}
std::filesystem::path defaultTempDir()

View File

@@ -5,5 +5,4 @@ include_dirs += include_directories('../..')
headers += files(
'signals-impl.hh',
'windows-async-pipe.hh',
'windows-error.hh',
)

View File

@@ -1,54 +0,0 @@
#pragma once
///@file
#ifdef _WIN32
# include <errhandlingapi.h>
# include "nix/util/error.hh"
namespace nix::windows {
/**
* Windows Error type.
*
* Unless you need to catch a specific error number, don't catch this in
* portable code. Catch `SystemError` instead.
*/
class WinError : public SystemError
{
public:
DWORD lastError;
/**
* Construct using the explicitly-provided error number.
* `FormatMessageA` will be used to try to add additional
* information to the message.
*/
template<typename... Args>
WinError(DWORD lastError, const Args &... args)
: SystemError("")
, lastError(lastError)
{
auto hf = HintFmt(args...);
err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), renderError(lastError));
}
/**
* Construct using `GetLastError()` and the ambient "last error".
*
* Be sure to not perform another last-error-modifying operation
* before calling this constructor!
*/
template<typename... Args>
WinError(const Args &... args)
: WinError(GetLastError(), args...)
{
}
private:
std::string renderError(DWORD lastError);
};
} // namespace nix::windows
#endif

View File

@@ -1,6 +1,5 @@
#ifdef _WIN32
# include <ioapiset.h>
# include "nix/util/windows-error.hh"
# include "nix/util/logging.hh"
# include "nix/util/util.hh"

View File

@@ -10,7 +10,6 @@
#include "nix/util/serialise.hh"
#include "nix/util/file-system.hh"
#include "nix/util/util.hh"
#include "nix/util/windows-error.hh"
#include <cerrno>
#include <cstdlib>

View File

@@ -2,7 +2,6 @@
#include "nix/util/users.hh"
#include "nix/util/environment-variables.hh"
#include "nix/util/file-system.hh"
#include "nix/util/windows-error.hh"
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN

View File

@@ -2,7 +2,6 @@
#ifdef _WIN32
# include "nix/util/windows-async-pipe.hh"
# include "nix/util/windows-error.hh"
namespace nix::windows {

View File

@@ -1,5 +1,7 @@
#ifdef _WIN32
# include "nix/util/windows-error.hh"
# include "nix/util/error.hh"
# include <error.h>
# define WIN32_LEAN_AND_MEAN
# include <windows.h>

View File

@@ -38,7 +38,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
if (!namePart)
namePart = baseNameOf(path);
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
auto sourcePath = makeFSSourceAccessor(makeParentCanonical(path));
auto storePath = dryRun ? store->computeStorePath(*namePart, sourcePath, caMethod, hashAlgo, {}).first
: store->addToStoreSlow(*namePart, sourcePath, caMethod, hashAlgo, {}).path;

View File

@@ -85,9 +85,7 @@ struct CmdHashBase : Command
return std::make_unique<HashSink>(hashAlgo);
};
auto makeSourcePath = [&]() -> SourcePath {
return PosixSourceAccessor::createAtRoot(makeParentCanonical(path));
};
auto makeSourcePath = [&]() -> SourcePath { return makeFSSourceAccessor(makeParentCanonical(path)); };
Hash h{HashAlgorithm::SHA256}; // throwaway def to appease C++
switch (mode) {

View File

@@ -191,7 +191,7 @@ static void opAdd(Strings opFlags, Strings opArgs)
throw UsageError("unknown flag");
for (auto & i : opArgs) {
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
auto sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), sourcePath)));
}
}
@@ -215,7 +215,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
opArgs.pop_front();
for (auto & i : opArgs) {
auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i));
auto sourcePath = makeFSSourceAccessor(makeParentCanonical(i));
std::cout << fmt(
"%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), sourcePath, method, hashAlgo).path));
}

View File

@@ -256,7 +256,7 @@ hashPath(char * algo, int base32, char * path)
PPCODE:
try {
Hash h = hashPath(
PosixSourceAccessor::createAtRoot(path),
makeFSSourceAccessor(absPath(Path(path))),
FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first;
auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
@@ -336,7 +336,7 @@ StoreWrapper::addToStore(char * srcPath, int recursive, char * algo)
auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat;
auto path = THIS->store->addToStore(
std::string(baseNameOf(srcPath)),
PosixSourceAccessor::createAtRoot(srcPath),
makeFSSourceAccessor(absPath(Path(srcPath))),
method, parseHashAlgo(algo));
XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0)));
} catch (Error & e) {

View File

@@ -0,0 +1,16 @@
error:
… while evaluating the attribute 'absolutePath'
at /pwd/lang/eval-fail-readDir-nonexistent-1.nix:2:3:
1| {
2| absolutePath = builtins.readDir /this/path/really/should/not/exist;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-nonexistent-1.nix:2:18:
1| {
2| absolutePath = builtins.readDir /this/path/really/should/not/exist;
| ^
3| }
error: path '/this/path/really/should/not/exist' does not exist

View File

@@ -0,0 +1,3 @@
{
absolutePath = builtins.readDir /this/path/really/should/not/exist;
}

View File

@@ -0,0 +1,16 @@
error:
… while evaluating the attribute 'relativePath'
at /pwd/lang/eval-fail-readDir-nonexistent-2.nix:2:3:
1| {
2| relativePath = builtins.readDir ./this/path/really/should/not/exist;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-nonexistent-2.nix:2:18:
1| {
2| relativePath = builtins.readDir ./this/path/really/should/not/exist;
| ^
3| }
error: path '/pwd/lang/this/path/really/should/not/exist' does not exist

View File

@@ -0,0 +1,3 @@
{
relativePath = builtins.readDir ./this/path/really/should/not/exist;
}

View File

@@ -0,0 +1,16 @@
error:
… while evaluating the attribute 'regularFile'
at /pwd/lang/eval-fail-readDir-not-a-directory-1.nix:2:3:
1| {
2| regularFile = builtins.readDir ./readDir/bar;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-not-a-directory-1.nix:2:17:
1| {
2| regularFile = builtins.readDir ./readDir/bar;
| ^
3| }
error: '/pwd/lang/readDir/bar' is not a directory

View File

@@ -0,0 +1,3 @@
{
regularFile = builtins.readDir ./readDir/bar;
}

View File

@@ -0,0 +1,16 @@
error:
… while evaluating the attribute 'symlinkedRegularFile'
at /pwd/lang/eval-fail-readDir-not-a-directory-2.nix:2:3:
1| {
2| symlinkedRegularFile = builtins.readDir ./readDir/linked;
| ^
3| }
… while calling the 'readDir' builtin
at /pwd/lang/eval-fail-readDir-not-a-directory-2.nix:2:26:
1| {
2| symlinkedRegularFile = builtins.readDir ./readDir/linked;
| ^
3| }
error: '/pwd/lang/readDir/foo/git-hates-directories' is not a directory

View File

@@ -0,0 +1,3 @@
{
symlinkedRegularFile = builtins.readDir ./readDir/linked;
}

View File

@@ -0,0 +1 @@
{ git-hates-directories = "regular"; }

View File

@@ -0,0 +1 @@
builtins.readDir ./readDir/ldir