Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1aac1884c5 | ||
|
|
ae9e8a178f | ||
|
|
8054e81351 | ||
|
|
26378d285b | ||
|
|
8449b3cac3 | ||
|
|
cabf1d9b40 | ||
|
|
4972085195 | ||
|
|
0bda735c57 | ||
|
|
54ac8fa919 | ||
|
|
b7166df536 | ||
|
|
0a8a8538b1 | ||
|
|
a5199bd826 | ||
|
|
33656097e4 | ||
|
|
567636cce8 | ||
|
|
6717482eaf |
No files matched your search
@@ -28,7 +28,7 @@
|
||||
<key>SoftResourceLimits</key>
|
||||
<dict>
|
||||
<key>NumberOfFiles</key>
|
||||
<integer>4096</integer>
|
||||
<integer>1048576</integer>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -9,7 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
|
||||
[Service]
|
||||
ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
|
||||
KillMode=process
|
||||
LimitNOFILE=4096
|
||||
LimitNOFILE=1048576
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "gc-store.hh"
|
||||
#include "util.hh"
|
||||
#include "loggers.hh"
|
||||
#include "progress-bar.hh"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
@@ -410,6 +411,8 @@ RunPager::RunPager()
|
||||
if (!pager) pager = getenv("PAGER");
|
||||
if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return;
|
||||
|
||||
stopProgressBar();
|
||||
|
||||
Pipe toPager;
|
||||
toPager.create();
|
||||
|
||||
|
||||
@@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised()
|
||||
/* Add the relevant output closures of the input derivation
|
||||
`i' as input paths. Only add the closures of output paths
|
||||
that are specified as inputs. */
|
||||
for (auto & j : wantedDepOutputs)
|
||||
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
|
||||
for (auto & j : wantedDepOutputs) {
|
||||
/* TODO (impure derivations-induced tech debt):
|
||||
Tracking input derivation outputs statefully through the
|
||||
goals is error prone and has led to bugs.
|
||||
For a robust nix, we need to move towards the `else` branch,
|
||||
which does not rely on goal state to match up with the
|
||||
reality of the store, which is our real source of truth.
|
||||
However, the impure derivations feature still relies on this
|
||||
fragile way of doing things, because its builds do not have
|
||||
a representation in the store, which is a usability problem
|
||||
in itself */
|
||||
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
|
||||
worker.store.computeFSClosure(*outPath, inputPaths);
|
||||
else
|
||||
throw Error(
|
||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
}
|
||||
else {
|
||||
auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
|
||||
auto outMapPath = outMap.find(j);
|
||||
if (outMapPath == outMap.end()) {
|
||||
throw Error(
|
||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
|
||||
}
|
||||
worker.store.computeFSClosure(outMapPath->second, inputPaths);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ public:
|
||||
void init() override
|
||||
{
|
||||
// FIXME: do this lazily?
|
||||
if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
|
||||
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
|
||||
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
|
||||
priority.setDefault(cacheInfo->priority);
|
||||
} else {
|
||||
|
||||
@@ -84,11 +84,10 @@ public:
|
||||
|
||||
Sync<State> _state;
|
||||
|
||||
NarInfoDiskCacheImpl()
|
||||
NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite")
|
||||
{
|
||||
auto state(_state.lock());
|
||||
|
||||
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
|
||||
createDirs(dirOf(dbPath));
|
||||
|
||||
state->db = SQLite(dbPath);
|
||||
@@ -98,7 +97,7 @@ public:
|
||||
state->db.exec(schema);
|
||||
|
||||
state->insertCache.create(state->db,
|
||||
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
|
||||
"insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;");
|
||||
|
||||
state->queryCache.create(state->db,
|
||||
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?");
|
||||
@@ -166,20 +165,61 @@ public:
|
||||
return i->second;
|
||||
}
|
||||
|
||||
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
|
||||
private:
|
||||
|
||||
std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
|
||||
{
|
||||
retrySQLite<void>([&]() {
|
||||
auto i = state.caches.find(uri);
|
||||
if (i == state.caches.end()) {
|
||||
auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
|
||||
if (!queryCache.next())
|
||||
return std::nullopt;
|
||||
auto cache = Cache {
|
||||
.id = (int) queryCache.getInt(0),
|
||||
.storeDir = queryCache.getStr(1),
|
||||
.wantMassQuery = queryCache.getInt(2) != 0,
|
||||
.priority = (int) queryCache.getInt(3),
|
||||
};
|
||||
state.caches.emplace(uri, cache);
|
||||
}
|
||||
return getCache(state, uri);
|
||||
}
|
||||
|
||||
public:
|
||||
int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
|
||||
{
|
||||
return retrySQLite<int>([&]() {
|
||||
auto state(_state.lock());
|
||||
SQLiteTxn txn(state->db);
|
||||
|
||||
// FIXME: race
|
||||
// To avoid the race, we have to check if maybe someone hasn't yet created
|
||||
// the cache for this URI in the meantime.
|
||||
auto cache(queryCacheRaw(*state, uri));
|
||||
|
||||
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
|
||||
assert(sqlite3_changes(state->db) == 1);
|
||||
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
|
||||
if (cache)
|
||||
return cache->id;
|
||||
|
||||
Cache ret {
|
||||
.id = -1, // set below
|
||||
.storeDir = storeDir,
|
||||
.wantMassQuery = wantMassQuery,
|
||||
.priority = priority,
|
||||
};
|
||||
|
||||
{
|
||||
auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority));
|
||||
assert(r.next());
|
||||
ret.id = (int) r.getInt(0);
|
||||
}
|
||||
|
||||
state->caches[uri] = ret;
|
||||
|
||||
txn.commit();
|
||||
return ret.id;
|
||||
});
|
||||
}
|
||||
|
||||
std::optional<CacheInfo> cacheExists(const std::string & uri) override
|
||||
std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override
|
||||
{
|
||||
return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
|
||||
auto state(_state.lock());
|
||||
@@ -196,6 +236,7 @@ public:
|
||||
auto & cache(getCache(*state, uri));
|
||||
|
||||
return CacheInfo {
|
||||
.id = cache.id,
|
||||
.wantMassQuery = cache.wantMassQuery,
|
||||
.priority = cache.priority
|
||||
};
|
||||
@@ -359,4 +400,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache()
|
||||
return cache;
|
||||
}
|
||||
|
||||
ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath)
|
||||
{
|
||||
return make_ref<NarInfoDiskCacheImpl>(dbPath);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -13,16 +13,17 @@ public:
|
||||
|
||||
virtual ~NarInfoDiskCache() { }
|
||||
|
||||
virtual void createCache(const std::string & uri, const Path & storeDir,
|
||||
virtual int createCache(const std::string & uri, const Path & storeDir,
|
||||
bool wantMassQuery, int priority) = 0;
|
||||
|
||||
struct CacheInfo
|
||||
{
|
||||
int id;
|
||||
bool wantMassQuery;
|
||||
int priority;
|
||||
};
|
||||
|
||||
virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
|
||||
virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0;
|
||||
|
||||
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
|
||||
const std::string & uri, const std::string & hashPart) = 0;
|
||||
@@ -45,4 +46,6 @@ public:
|
||||
multiple threads. */
|
||||
ref<NarInfoDiskCache> getNarInfoDiskCache();
|
||||
|
||||
ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath);
|
||||
|
||||
}
|
||||
@@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
|
||||
|
||||
void init() override
|
||||
{
|
||||
if (auto cacheInfo = diskCache->cacheExists(getUri())) {
|
||||
if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
|
||||
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
|
||||
priority.setDefault(cacheInfo->priority);
|
||||
} else {
|
||||
|
||||
@@ -36,6 +36,15 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor
|
||||
throw SQLiteError(path, err, exterr, std::move(hf));
|
||||
}
|
||||
|
||||
static void traceSQL(void * x, const char * sql)
|
||||
{
|
||||
// wacky delimiters:
|
||||
// so that we're quite unambiguous without escaping anything
|
||||
// notice instead of trace:
|
||||
// so that this can be enabled without getting the firehose in our face.
|
||||
notice("SQL<[%1%]>", sql);
|
||||
};
|
||||
|
||||
SQLite::SQLite(const Path & path, bool create)
|
||||
{
|
||||
// useSQLiteWAL also indicates what virtual file system we need. Using
|
||||
@@ -49,6 +58,11 @@ SQLite::SQLite(const Path & path, bool create)
|
||||
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
|
||||
SQLiteError::throw_(db, "setting timeout");
|
||||
|
||||
if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") {
|
||||
// To debug sqlite statements; trace all of them
|
||||
sqlite3_trace(db, &traceSQL, nullptr);
|
||||
}
|
||||
|
||||
exec("pragma foreign_keys = 1");
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
#include "nar-info-disk-cache.hh"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include "sqlite.hh"
|
||||
#include <sqlite3.h>
|
||||
|
||||
|
||||
namespace nix {
|
||||
|
||||
TEST(NarInfoDiskCacheImpl, create_and_read) {
|
||||
// This is a large single test to avoid some setup overhead.
|
||||
|
||||
int prio = 12345;
|
||||
bool wantMassQuery = true;
|
||||
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete delTmpDir(tmpDir);
|
||||
Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite");
|
||||
|
||||
int savedId;
|
||||
int barId;
|
||||
SQLite db;
|
||||
SQLiteStmt getIds;
|
||||
|
||||
{
|
||||
auto cache = getTestNarInfoDiskCache(dbPath);
|
||||
|
||||
// Set up "background noise" and check that different caches receive different ids
|
||||
{
|
||||
auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio);
|
||||
auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12);
|
||||
ASSERT_NE(bc1, bc2);
|
||||
barId = bc1;
|
||||
}
|
||||
|
||||
// Check that the fields are saved and returned correctly. This does not test
|
||||
// the select statement yet, because of in-memory caching.
|
||||
savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);;
|
||||
{
|
||||
auto r = cache->upToDateCacheExists("http://foo");
|
||||
ASSERT_TRUE(r);
|
||||
ASSERT_EQ(r->priority, prio);
|
||||
ASSERT_EQ(r->wantMassQuery, wantMassQuery);
|
||||
ASSERT_EQ(savedId, r->id);
|
||||
}
|
||||
|
||||
// We're going to pay special attention to the id field because we had a bug
|
||||
// that changed it.
|
||||
db = SQLite(dbPath);
|
||||
getIds.create(db, "select id from BinaryCaches where url = 'http://foo'");
|
||||
|
||||
{
|
||||
auto q(getIds.use());
|
||||
ASSERT_TRUE(q.next());
|
||||
ASSERT_EQ(savedId, q.getInt(0));
|
||||
ASSERT_FALSE(q.next());
|
||||
}
|
||||
|
||||
// Pretend that the caches are older, but keep one up to date, as "background noise"
|
||||
db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';");
|
||||
|
||||
// This shows that the in-memory cache works
|
||||
{
|
||||
auto r = cache->upToDateCacheExists("http://foo");
|
||||
ASSERT_TRUE(r);
|
||||
ASSERT_EQ(r->priority, prio);
|
||||
ASSERT_EQ(r->wantMassQuery, wantMassQuery);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// We can't clear the in-memory cache, so we use a new cache object. This is
|
||||
// more realistic anyway.
|
||||
auto cache2 = getTestNarInfoDiskCache(dbPath);
|
||||
|
||||
{
|
||||
auto r = cache2->upToDateCacheExists("http://foo");
|
||||
ASSERT_FALSE(r);
|
||||
}
|
||||
|
||||
// "Update", same data, check that the id number is reused
|
||||
cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);
|
||||
|
||||
{
|
||||
auto r = cache2->upToDateCacheExists("http://foo");
|
||||
ASSERT_TRUE(r);
|
||||
ASSERT_EQ(r->priority, prio);
|
||||
ASSERT_EQ(r->wantMassQuery, wantMassQuery);
|
||||
ASSERT_EQ(r->id, savedId);
|
||||
}
|
||||
|
||||
{
|
||||
auto q(getIds.use());
|
||||
ASSERT_TRUE(q.next());
|
||||
auto currentId = q.getInt(0);
|
||||
ASSERT_FALSE(q.next());
|
||||
ASSERT_EQ(currentId, savedId);
|
||||
}
|
||||
|
||||
// Check that the fields can be modified, and the id remains the same
|
||||
{
|
||||
auto r0 = cache2->upToDateCacheExists("https://bar");
|
||||
ASSERT_FALSE(r0);
|
||||
|
||||
cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10);
|
||||
auto r = cache2->upToDateCacheExists("https://bar");
|
||||
ASSERT_EQ(r->wantMassQuery, !wantMassQuery);
|
||||
ASSERT_EQ(r->priority, prio + 10);
|
||||
ASSERT_EQ(r->id, barId);
|
||||
}
|
||||
|
||||
// // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp)
|
||||
// {
|
||||
// cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20);
|
||||
// auto r = cache2->upToDateCacheExists("https://bar");
|
||||
// ASSERT_EQ(r->wantMassQuery, wantMassQuery);
|
||||
// ASSERT_EQ(r->priority, prio + 20);
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -70,3 +70,54 @@ testNormalization () {
|
||||
}
|
||||
|
||||
testNormalization
|
||||
|
||||
# https://github.com/NixOS/nix/issues/6572
|
||||
issue_6572_independent_outputs() {
|
||||
nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json
|
||||
|
||||
# Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation.
|
||||
p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
|
||||
nix-store --delete "$p" # Clean up for next test
|
||||
|
||||
# Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
|
||||
nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)"
|
||||
p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
|
||||
cmp $p <<EOF
|
||||
first
|
||||
second
|
||||
EOF
|
||||
nix-store --delete "$p" # Clean up for next test
|
||||
|
||||
# Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
|
||||
nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.second)"
|
||||
p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
|
||||
cmp $p <<EOF
|
||||
first
|
||||
second
|
||||
EOF
|
||||
nix-store --delete "$p" # Clean up for next test
|
||||
}
|
||||
issue_6572_independent_outputs
|
||||
|
||||
|
||||
# https://github.com/NixOS/nix/issues/6572
|
||||
issue_6572_dependent_outputs() {
|
||||
|
||||
nix build -f multiple-outputs.nix --json a --no-link > $TEST_ROOT/a.json
|
||||
|
||||
# # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation.
|
||||
p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths)
|
||||
nix-store --delete "$p" # Clean up for next test
|
||||
|
||||
# Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
|
||||
nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)"
|
||||
p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths)
|
||||
cmp $p <<EOF
|
||||
first
|
||||
second
|
||||
EOF
|
||||
nix-store --delete "$p" # Clean up for next test
|
||||
}
|
||||
if isDaemonNewer "2.12pre0"; then
|
||||
issue_6572_dependent_outputs
|
||||
fi
|
||||
@@ -31,6 +31,15 @@ rec {
|
||||
helloString = "Hello, world!";
|
||||
};
|
||||
|
||||
use-a = mkDerivation {
|
||||
name = "use-a";
|
||||
inherit (a) first second;
|
||||
builder = builtins.toFile "builder.sh"
|
||||
''
|
||||
cat $first/file $second/file >$out
|
||||
'';
|
||||
};
|
||||
|
||||
b = mkDerivation {
|
||||
defaultOutput = assert a.second.helloString == "Hello, world!"; a;
|
||||
firstOutput = assert a.outputName == "first"; a.first.first;
|
||||
@@ -87,4 +96,25 @@ rec {
|
||||
buildCommand = "mkdir $a $b $c";
|
||||
};
|
||||
|
||||
independent = mkDerivation {
|
||||
name = "multiple-outputs-independent";
|
||||
outputs = [ "first" "second" ];
|
||||
builder = builtins.toFile "builder.sh"
|
||||
''
|
||||
mkdir $first $second
|
||||
test -z $all
|
||||
echo "first" > $first/file
|
||||
echo "second" > $second/file
|
||||
'';
|
||||
};
|
||||
|
||||
use-independent = mkDerivation {
|
||||
name = "use-independent";
|
||||
inherit (a) first second;
|
||||
builder = builtins.toFile "builder.sh"
|
||||
''
|
||||
cat $first/file $second/file >$out
|
||||
'';
|
||||
};
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user