Compare commits

...
15 Commits
Author SHA1 Message Date
Théophane Hufschmitt 1aac1884c5 Merge pull request #7306 from NixOS/backport-7283-to-2.11-maintenance
[Backport 2.11-maintenance] Fix #6572 `requires non-existent output`
2023-03-03 13:43:01 +01:00
Théophane Hufschmitt ae9e8a178f Merge pull request #7033 from NixOS/backport-7032-to-2.11-maintenance
[Backport 2.11-maintenance] RunPager: Stop the progress bar
2023-03-03 13:40:00 +01:00
Robert Hensing 8054e81351 Merge pull request #7831 from hercules-ci/backport-7616-to-2.11-maintenance
Backport 7616 to 2.11 maintenance
2023-02-17 14:09:25 +01:00
Robert Hensing 26378d285b NarInfoDiskCache: Also test id consistency with updated fields
And clarify test

(cherry picked from commit 19b495a48a)
2023-02-14 14:45:29 +01:00
Robert Hensing 8449b3cac3 NarInfoDiskCache: Keep BinaryCache.id stable and improve test
Fixes #3898

The entire `BinaryCaches` row used to get replaced after it became
stale according to the `timestamp` column. In a concurrent scenario,
this leads to foreign key conflicts as different instances of the
in-process `state.caches` cache now differ, with the consequence that
the older process still tries to use the `id` number of the old record.

Furthermore, this phenomenon appears to have caused the cache for
actual narinfos to be erased about every week, while the default
ttl for narinfos was supposed to be 30 days.

(cherry picked from commit fb94d5cabd)
2023-02-14 14:45:29 +01:00
Robert Hensing cabf1d9b40 NarInfoDiskCache: Prepare reproducer for #3898
(cherry picked from commit 2ceece3ef3)
2023-02-14 14:45:29 +01:00
Robert Hensing 4972085195 NarInfoDiskCacheImpl: Make dbPath a parameter
This allows testing with a clean database.

(cherry picked from commit 79f62d2dda)
2023-02-13 19:22:27 +01:00
Robert Hensing 0bda735c57 NarInfoDiskCache: Rename cacheExists -> upToDateCacheExists
This is slightly more accurate considering that an outdated record
may exist in the persistent cache. Possibly-outdated records are
quite relevant as they may be foreign keys to more recent information
that we want to keep, but we will not return them here.

(cherry picked from commit 29f0b196f4)
2023-02-13 19:22:16 +01:00
Robert Hensing 54ac8fa919 sqlite.cc: Add SQL tracing
Set environment variable NIX_DEBUG_SQLITE_TRACES=1 to log all sql statements.

(cherry picked from commit 8a0ef5d58e)
2023-02-13 19:21:50 +01:00
Eelco Dolstra b7166df536 Merge pull request #7432 from NixOS/backport-6645-to-2.11-maintenance
[Backport 2.11-maintenance] systemd,launchd match nixos open files limit
2022-12-09 17:26:40 +01:00
Artturin 0a8a8538b1 systemd,launchd match nixos open files limit
it was bumped in https://github.com/NixOS/nixpkgs/pull/176558

(cherry picked from commit 2320a2f93e)
2022-12-09 12:36:32 +00:00
Robert Hensing a5199bd826 tests: Reproduce #6572
(cherry picked from commit c279ddb18c)
2022-11-15 15:24:52 +00:00
Robert Hensing 33656097e4 derivation-goal: Fix requires non-existing output error
It occurred when a output of the dependency was already available,
so it didn't need rebuilding and didn't get added to the
inputDrvOutputs.
This process-related info wasn't suitable for the purpose of finding
the actual input paths for the builder. It is better to do this in
absolute terms by querying the store.

(cherry picked from commit 7e162c69fe)
2022-11-15 15:24:52 +00:00
Eelco Dolstra 567636cce8 Bump version 2022-09-15 19:12:08 +02:00
Eelco Dolstra 6717482eaf RunPager: Stop the progress bar
In particular, the progress bar was interfering with 'less' rendering
in '--help' (e.g. run 'nix --help' and hit '/' to search).

(cherry picked from commit c6ff33ff5c)
2022-09-13 14:58:01 +00:00
13 changed files with 310 additions and 23 deletions

No files matched your search

+1 -1
View File
@@ -1 +1 @@
2.11.1
2.11.2
+1 -1
View File
@@ -28,7 +28,7 @@
<key>SoftResourceLimits</key>
<dict>
<key>NumberOfFiles</key>
<integer>4096</integer>
<integer>1048576</integer>
</dict>
</dict>
</plist>
+1 -1
View File
@@ -9,7 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
[Service]
ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
KillMode=process
LimitNOFILE=4096
LimitNOFILE=1048576
[Install]
WantedBy=multi-user.target
+3
View File
@@ -4,6 +4,7 @@
#include "gc-store.hh"
#include "util.hh"
#include "loggers.hh"
#include "progress-bar.hh"
#include <algorithm>
#include <cctype>
@@ -410,6 +411,8 @@ RunPager::RunPager()
if (!pager) pager = getenv("PAGER");
if (pager && ((std::string) pager == "" || (std::string) pager == "cat")) return;
stopProgressBar();
Pipe toPager;
toPager.create();
+24 -6
View File
@@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
for (auto & j : wantedDepOutputs)
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
for (auto & j : wantedDepOutputs) {
/* TODO (impure derivations-induced tech debt):
Tracking input derivation outputs statefully through the
goals is error prone and has led to bugs.
For a robust nix, we need to move towards the `else` branch,
which does not rely on goal state to match up with the
reality of the store, which is our real source of truth.
However, the impure derivations feature still relies on this
fragile way of doing things, because its builds do not have
a representation in the store, which is a usability problem
in itself */
if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
worker.store.computeFSClosure(*outPath, inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
}
else {
auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
auto outMapPath = outMap.find(j);
if (outMapPath == outMap.end()) {
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
}
worker.store.computeFSClosure(outMapPath->second, inputPaths);
}
}
}
}
+1 -1
View File
@@ -56,7 +56,7 @@ public:
void init() override
{
// FIXME: do this lazily?
if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
} else {
+56 -10
View File
@@ -84,11 +84,10 @@ public:
Sync<State> _state;
NarInfoDiskCacheImpl()
NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite")
{
auto state(_state.lock());
Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
createDirs(dirOf(dbPath));
state->db = SQLite(dbPath);
@@ -98,7 +97,7 @@ public:
state->db.exec(schema);
state->insertCache.create(state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
"insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;");
state->queryCache.create(state->db,
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?");
@@ -166,20 +165,61 @@ public:
return i->second;
}
void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
private:
std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
{
retrySQLite<void>([&]() {
auto i = state.caches.find(uri);
if (i == state.caches.end()) {
auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
if (!queryCache.next())
return std::nullopt;
auto cache = Cache {
.id = (int) queryCache.getInt(0),
.storeDir = queryCache.getStr(1),
.wantMassQuery = queryCache.getInt(2) != 0,
.priority = (int) queryCache.getInt(3),
};
state.caches.emplace(uri, cache);
}
return getCache(state, uri);
}
public:
int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
{
return retrySQLite<int>([&]() {
auto state(_state.lock());
SQLiteTxn txn(state->db);
// FIXME: race
// To avoid the race, we have to check if maybe someone hasn't yet created
// the cache for this URI in the meantime.
auto cache(queryCacheRaw(*state, uri));
state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
if (cache)
return cache->id;
Cache ret {
.id = -1, // set below
.storeDir = storeDir,
.wantMassQuery = wantMassQuery,
.priority = priority,
};
{
auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority));
assert(r.next());
ret.id = (int) r.getInt(0);
}
state->caches[uri] = ret;
txn.commit();
return ret.id;
});
}
std::optional<CacheInfo> cacheExists(const std::string & uri) override
std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override
{
return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
auto state(_state.lock());
@@ -196,6 +236,7 @@ public:
auto & cache(getCache(*state, uri));
return CacheInfo {
.id = cache.id,
.wantMassQuery = cache.wantMassQuery,
.priority = cache.priority
};
@@ -359,4 +400,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache()
return cache;
}
ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath)
{
return make_ref<NarInfoDiskCacheImpl>(dbPath);
}
}
+5 -2
View File
@@ -13,16 +13,17 @@ public:
virtual ~NarInfoDiskCache() { }
virtual void createCache(const std::string & uri, const Path & storeDir,
virtual int createCache(const std::string & uri, const Path & storeDir,
bool wantMassQuery, int priority) = 0;
struct CacheInfo
{
int id;
bool wantMassQuery;
int priority;
};
virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const std::string & hashPart) = 0;
@@ -45,4 +46,6 @@ public:
multiple threads. */
ref<NarInfoDiskCache> getNarInfoDiskCache();
ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath);
}
+1 -1
View File
@@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
void init() override
{
if (auto cacheInfo = diskCache->cacheExists(getUri())) {
if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
wantMassQuery.setDefault(cacheInfo->wantMassQuery);
priority.setDefault(cacheInfo->priority);
} else {
+14
View File
@@ -36,6 +36,15 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor
throw SQLiteError(path, err, exterr, std::move(hf));
}
static void traceSQL(void * x, const char * sql)
{
// wacky delimiters:
// so that we're quite unambiguous without escaping anything
// notice instead of trace:
// so that this can be enabled without getting the firehose in our face.
notice("SQL<[%1%]>", sql);
};
SQLite::SQLite(const Path & path, bool create)
{
// useSQLiteWAL also indicates what virtual file system we need. Using
@@ -49,6 +58,11 @@ SQLite::SQLite(const Path & path, bool create)
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
SQLiteError::throw_(db, "setting timeout");
if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") {
// To debug sqlite statements; trace all of them
sqlite3_trace(db, &traceSQL, nullptr);
}
exec("pragma foreign_keys = 1");
}
+122
View File
@@ -0,0 +1,122 @@
#include "nar-info-disk-cache.hh"
#include <gtest/gtest.h>
#include "sqlite.hh"
#include <sqlite3.h>
namespace nix {
TEST(NarInfoDiskCacheImpl, create_and_read) {
// This is a large single test to avoid some setup overhead.
int prio = 12345;
bool wantMassQuery = true;
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir);
Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite");
int savedId;
int barId;
SQLite db;
SQLiteStmt getIds;
{
auto cache = getTestNarInfoDiskCache(dbPath);
// Set up "background noise" and check that different caches receive different ids
{
auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio);
auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12);
ASSERT_NE(bc1, bc2);
barId = bc1;
}
// Check that the fields are saved and returned correctly. This does not test
// the select statement yet, because of in-memory caching.
savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);;
{
auto r = cache->upToDateCacheExists("http://foo");
ASSERT_TRUE(r);
ASSERT_EQ(r->priority, prio);
ASSERT_EQ(r->wantMassQuery, wantMassQuery);
ASSERT_EQ(savedId, r->id);
}
// We're going to pay special attention to the id field because we had a bug
// that changed it.
db = SQLite(dbPath);
getIds.create(db, "select id from BinaryCaches where url = 'http://foo'");
{
auto q(getIds.use());
ASSERT_TRUE(q.next());
ASSERT_EQ(savedId, q.getInt(0));
ASSERT_FALSE(q.next());
}
// Pretend that the caches are older, but keep one up to date, as "background noise"
db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';");
// This shows that the in-memory cache works
{
auto r = cache->upToDateCacheExists("http://foo");
ASSERT_TRUE(r);
ASSERT_EQ(r->priority, prio);
ASSERT_EQ(r->wantMassQuery, wantMassQuery);
}
}
{
// We can't clear the in-memory cache, so we use a new cache object. This is
// more realistic anyway.
auto cache2 = getTestNarInfoDiskCache(dbPath);
{
auto r = cache2->upToDateCacheExists("http://foo");
ASSERT_FALSE(r);
}
// "Update", same data, check that the id number is reused
cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);
{
auto r = cache2->upToDateCacheExists("http://foo");
ASSERT_TRUE(r);
ASSERT_EQ(r->priority, prio);
ASSERT_EQ(r->wantMassQuery, wantMassQuery);
ASSERT_EQ(r->id, savedId);
}
{
auto q(getIds.use());
ASSERT_TRUE(q.next());
auto currentId = q.getInt(0);
ASSERT_FALSE(q.next());
ASSERT_EQ(currentId, savedId);
}
// Check that the fields can be modified, and the id remains the same
{
auto r0 = cache2->upToDateCacheExists("https://bar");
ASSERT_FALSE(r0);
cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10);
auto r = cache2->upToDateCacheExists("https://bar");
ASSERT_EQ(r->wantMassQuery, !wantMassQuery);
ASSERT_EQ(r->priority, prio + 10);
ASSERT_EQ(r->id, barId);
}
// // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp)
// {
// cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20);
// auto r = cache2->upToDateCacheExists("https://bar");
// ASSERT_EQ(r->wantMassQuery, wantMassQuery);
// ASSERT_EQ(r->priority, prio + 20);
// }
}
}
}
+51
View File
@@ -70,3 +70,54 @@ testNormalization () {
}
testNormalization
# https://github.com/NixOS/nix/issues/6572
issue_6572_independent_outputs() {
nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json
# Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation.
p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
nix-store --delete "$p" # Clean up for next test
# Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)"
p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
cmp $p <<EOF
first
second
EOF
nix-store --delete "$p" # Clean up for next test
# Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.second)"
p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths)
cmp $p <<EOF
first
second
EOF
nix-store --delete "$p" # Clean up for next test
}
issue_6572_independent_outputs
# https://github.com/NixOS/nix/issues/6572
issue_6572_dependent_outputs() {
nix build -f multiple-outputs.nix --json a --no-link > $TEST_ROOT/a.json
# # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation.
p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths)
nix-store --delete "$p" # Clean up for next test
# Make sure that 'nix build' tracks input-outputs correctly when a single output is already present.
nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)"
p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths)
cmp $p <<EOF
first
second
EOF
nix-store --delete "$p" # Clean up for next test
}
if isDaemonNewer "2.12pre0"; then
issue_6572_dependent_outputs
fi
+30
View File
@@ -31,6 +31,15 @@ rec {
helloString = "Hello, world!";
};
use-a = mkDerivation {
name = "use-a";
inherit (a) first second;
builder = builtins.toFile "builder.sh"
''
cat $first/file $second/file >$out
'';
};
b = mkDerivation {
defaultOutput = assert a.second.helloString == "Hello, world!"; a;
firstOutput = assert a.outputName == "first"; a.first.first;
@@ -87,4 +96,25 @@ rec {
buildCommand = "mkdir $a $b $c";
};
independent = mkDerivation {
name = "multiple-outputs-independent";
outputs = [ "first" "second" ];
builder = builtins.toFile "builder.sh"
''
mkdir $first $second
test -z $all
echo "first" > $first/file
echo "second" > $second/file
'';
};
use-independent = mkDerivation {
name = "use-independent";
inherit (a) first second;
builder = builtins.toFile "builder.sh"
''
cat $first/file $second/file >$out
'';
};
}