Compare commits

...
Author SHA1 Message Date
Eelco Dolstra 3f53a435cb Factor out the local NAR cache into its own class 2026-01-14 22:22:44 +01:00
8 changed files with 121 additions and 49 deletions

No files matched your search

+3 -1
View File
@@ -13,6 +13,7 @@
#include "nix/util/callback.hh" #include "nix/util/callback.hh"
#include "nix/util/signals.hh" #include "nix/util/signals.hh"
#include "nix/util/archive.hh" #include "nix/util/archive.hh"
#include "nix/store/nar-cache.hh"
#include <chrono> #include <chrono>
#include <future> #include <future>
@@ -26,6 +27,7 @@ namespace nix {
BinaryCacheStore::BinaryCacheStore(Config & config) BinaryCacheStore::BinaryCacheStore(Config & config)
: config{config} : config{config}
, narCache{config.localNarCache.get() ? std::make_shared<NarCache>(*config.localNarCache.get()) : nullptr}
{ {
if (config.secretKeyFile != "") if (config.secretKeyFile != "")
signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)})); signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
@@ -552,7 +554,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
{ {
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache); return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
} }
ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath) ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)
@@ -12,6 +12,7 @@
namespace nix { namespace nix {
struct NarInfo; struct NarInfo;
class NarCache;
class RemoteFSAccessor; class RemoteFSAccessor;
struct BinaryCacheStoreConfig : virtual StoreConfig struct BinaryCacheStoreConfig : virtual StoreConfig
@@ -89,6 +90,8 @@ protected:
constexpr const static std::string cacheInfoFile = "nix-cache-info"; constexpr const static std::string cacheInfoFile = "nix-cache-info";
std::shared_ptr<NarCache> narCache;
BinaryCacheStore(Config &); BinaryCacheStore(Config &);
/** /**
@@ -56,6 +56,7 @@ headers = [ config_pub_h ] + files(
'machines.hh', 'machines.hh',
'make-content-addressed.hh', 'make-content-addressed.hh',
'names.hh', 'names.hh',
'nar-cache.hh',
'nar-info-disk-cache.hh', 'nar-info-disk-cache.hh',
'nar-info.hh', 'nar-info.hh',
'outputs-spec.hh', 'outputs-spec.hh',
@@ -0,0 +1,33 @@
#pragma once
#include "nix/util/hash.hh"
#include "nix/util/nar-accessor.hh"
#include <filesystem>
namespace nix {
class NarCache
{
const std::filesystem::path cacheDir;
std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
public:
NarCache(std::filesystem::path cacheDir);
void upsertNar(const Hash & narHash, Source & source);
void upsertNarListing(const Hash & narHash, std::string_view narListingData);
// FIXME: use a sink.
std::optional<std::string> getNar(const Hash & narHash);
// FIXME: use a sink.
GetNarBytes getNarBytes(const Hash & narHash);
std::optional<std::string> getNarListing(const Hash & narHash);
};
} // namespace nix
@@ -7,6 +7,8 @@
namespace nix { namespace nix {
struct NarCache;
class RemoteFSAccessor : public SourceAccessor class RemoteFSAccessor : public SourceAccessor
{ {
ref<Store> store; ref<Store> store;
@@ -25,7 +27,7 @@ class RemoteFSAccessor : public SourceAccessor
bool requireValidPath; bool requireValidPath;
std::optional<std::filesystem::path> cacheDir; std::shared_ptr<NarCache> narCache;
std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path); std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
@@ -38,8 +40,7 @@ public:
*/ */
std::shared_ptr<SourceAccessor> accessObject(const StorePath & path); std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
RemoteFSAccessor( RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::shared_ptr<NarCache> narCache = {});
ref<Store> store, bool requireValidPath = true, std::optional<std::filesystem::path> cacheDir = {});
std::optional<Stat> maybeLstat(const CanonPath & path) override; std::optional<Stat> maybeLstat(const CanonPath & path) override;
+1
View File
@@ -318,6 +318,7 @@ sources = files(
'make-content-addressed.cc', 'make-content-addressed.cc',
'misc.cc', 'misc.cc',
'names.cc', 'names.cc',
'nar-cache.cc',
'nar-info-disk-cache.cc', 'nar-info-disk-cache.cc',
'nar-info.cc', 'nar-info.cc',
'optimise-store.cc', 'optimise-store.cc',
+64
View File
@@ -0,0 +1,64 @@
#include "nix/store/nar-cache.hh"
#include "nix/util/file-system.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
namespace nix {
NarCache::NarCache(std::filesystem::path cacheDir_)
: cacheDir(std::move(cacheDir_))
{
assert(!cacheDir.empty());
createDirs(cacheDir);
}
std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext)
{
return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
}
void NarCache::upsertNar(const Hash & narHash, Source & source)
{
try {
/* FIXME: do this asynchronously. */
writeFile(makeCacheFile(narHash, "nar"), source);
} catch (SystemError &) {
ignoreExceptionExceptInterrupt();
}
}
void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData)
{
try {
writeFile(makeCacheFile(narHash, "ls"), narListingData);
} catch (SystemError &) {
ignoreExceptionExceptInterrupt();
}
}
std::optional<std::string> NarCache::getNar(const Hash & narHash)
{
try {
return nix::readFile(makeCacheFile(narHash, "nar"));
} catch (SystemError &) {
return std::nullopt;
}
}
GetNarBytes NarCache::getNarBytes(const Hash & narHash)
{
return seekableGetNarBytes(makeCacheFile(narHash, "nar"));
}
std::optional<std::string> NarCache::getNarListing(const Hash & narHash)
{
try {
return nix::readFile(makeCacheFile(narHash, "ls"));
} catch (SystemError &) {
return std::nullopt;
}
}
} // namespace nix
+12 -45
View File
@@ -1,21 +1,14 @@
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include "nix/store/remote-fs-accessor.hh" #include "nix/store/remote-fs-accessor.hh"
#include "nix/util/nar-accessor.hh" #include "nix/store/nar-cache.hh"
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
namespace nix { namespace nix {
RemoteFSAccessor::RemoteFSAccessor( RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, bool requireValidPath, std::shared_ptr<NarCache> narCache)
ref<Store> store, bool requireValidPath, std::optional<std::filesystem::path> cacheDir_)
: store(store) : store(store)
, requireValidPath(requireValidPath) , requireValidPath(requireValidPath)
, cacheDir(std::move(cacheDir_)) , narCache(std::move(narCache))
{ {
if (cacheDir)
createDirs(*cacheDir);
} }
std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) std::pair<ref<SourceAccessor>, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path)
@@ -47,48 +40,22 @@ std::shared_ptr<SourceAccessor> RemoteFSAccessor::accessObject(const StorePath &
return std::move(sink.s); return std::move(sink.s);
}; };
if (cacheDir) { if (narCache) {
auto makeCacheFile = [&](const std::string & ext) { if (auto listingData = narCache->getNarListing(info->narHash))
auto res = *cacheDir / info->narHash.to_string(HashFormat::Nix32, false); return cacheAccessor(makeLazyNarAccessor(
res += "."; nlohmann::json::parse(*listingData).template get<NarListing>(), narCache->getNarBytes(info->narHash)));
res += ext;
return res;
};
auto cacheFile = makeCacheFile("nar"); if (auto nar = narCache->getNar(info->narHash))
auto listingFile = makeCacheFile("ls"); return cacheAccessor(makeNarAccessor(std::move(*nar)));
if (nix::pathExists(cacheFile)) {
try {
return cacheAccessor(makeLazyNarAccessor(
nlohmann::json::parse(nix::readFile(listingFile)).template get<NarListing>(),
seekableGetNarBytes(cacheFile)));
} catch (SystemError &) {
}
try {
return cacheAccessor(makeNarAccessor(nix::readFile(cacheFile)));
} catch (SystemError &) {
}
}
auto nar = getNar(); auto nar = getNar();
try { StringSource source{nar};
/* FIXME: do this asynchronously. */ narCache->upsertNar(info->narHash, source);
writeFile(cacheFile, nar);
} catch (...) {
ignoreExceptionExceptInterrupt();
}
auto narAccessor = makeNarAccessor(std::move(nar)); auto narAccessor = makeNarAccessor(std::move(nar));
try { narCache->upsertNarListing(info->narHash, nlohmann::json(narAccessor->getListing()).dump());
nlohmann::json j = narAccessor->getListing();
writeFile(listingFile, j.dump());
} catch (...) {
ignoreExceptionExceptInterrupt();
}
return cacheAccessor(narAccessor); return cacheAccessor(narAccessor);
} }