Merge pull request #7877 from NixOS/backport-7856-to-2.12-maintenance

[Backport 2.12-maintenance] Wait with making /etc unwritable until after build env setup

.version

@@ -1 +1 @@
-2.12.0
+2.12.1

doc/manual/src/SUMMARY.md.in

@@ -64,7 +64,6 @@
   - [Hacking](contributing/hacking.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md)
   - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md)
   - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md)

flake.nix

@@ -15,7 +15,7 @@
         then ""
         else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}";
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
       linuxSystems = linux64BitSystems ++ [ "i686-linux" ];

misc/launchd/org.nixos.nix-daemon.plist.in

@@ -28,7 +28,7 @@
     <key>SoftResourceLimits</key>
     <dict>
       <key>NumberOfFiles</key>
-      <integer>4096</integer>
+      <integer>1048576</integer>
     </dict>
   </dict>
 </plist>

misc/systemd/nix-daemon.service.in

@@ -9,7 +9,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
 [Service]
 ExecStart=@@bindir@/nix-daemon nix-daemon --daemon
 KillMode=process
-LimitNOFILE=4096
+LimitNOFILE=1048576
 
 [Install]
 WantedBy=multi-user.target

scripts/install-multi-user.sh

@@ -578,7 +578,7 @@ EOF
     # to extract _just_ the user's note, instead it is prefixed with
     # some plist junk. This was causing the user note to always be set,
     # even if there was no reason for it.
-    if ! poly_user_note_get "$username" | grep -q "Nix build user $coreid"; then
+    if poly_user_note_get "$username" | grep -q "Nix build user $coreid"; then
         row "              Note" "Nix build user $coreid"
     else
         poly_user_note_set "$username" "Nix build user $coreid"

scripts/nix-profile-daemon.fish.in

@@ -1,3 +1,15 @@
+function add_path --argument-names new_path
+    if type -q fish_add_path
+        # fish 3.2.0 or newer
+	fish_add_path --prepend --global $new_path
+    else
+        # older versions of fish
+        if not contains $new_path $fish_user_paths
+            set --global fish_user_paths $new_path $fish_user_paths
+        end
+    end
+end
+
 # Only execute this file once per shell.
 if test -n "$__ETC_PROFILE_NIX_SOURCED"
   exit
@@ -31,5 +43,7 @@ else
   end
 end
 
-fish_add_path --prepend --global "@localstatedir@/nix/profiles/default/bin"
-fish_add_path --prepend --global "$HOME/.nix-profile/bin"
+add_path "@localstatedir@/nix/profiles/default/bin"
+add_path "$HOME/.nix-profile/bin"
+
+functions -e add_path

scripts/nix-profile.fish.in

@@ -1,3 +1,15 @@
+function add_path --argument-names new_path
+    if type -q fish_add_path
+        # fish 3.2.0 or newer
+	fish_add_path --prepend --global $new_path
+    else
+        # older versions of fish
+        if not contains $new_path $fish_user_paths
+            set --global fish_user_paths $new_path $fish_user_paths
+        end
+    end
+end
+
 if test -n "$HOME" && test -n "$USER"
 
     # Set up the per-user profile.
@@ -32,6 +44,8 @@ if test -n "$HOME" && test -n "$USER"
       set --export --prepend --path MANPATH "$NIX_LINK/share/man"
     end
 
-    fish_add_path --prepend --global "$NIX_LINK/bin"
+    add_path "$NIX_LINK/bin"
     set --erase NIX_LINK
 end
+
+functions -e add_path

src/libcmd/installables.cc

@@ -903,10 +903,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal
                                 DrvOutput outputId { *outputHash, output };
                                 auto realisation = store->queryRealisation(outputId);
                                 if (!realisation)
-                                    throw Error(
-                                        "cannot operate on an output of the "
-                                        "unbuilt derivation '%s'",
-                                        outputId.to_string());
+                                    throw MissingRealisation(outputId);
                                 outputs.insert_or_assign(output, realisation->outPath);
                             } else {
                                 // If ca-derivations isn't enabled, assume that

src/libstore/build/local-derivation-goal.cc

@@ -664,7 +664,8 @@ void LocalDerivationGoal::startBuilder()
            nobody account.  The latter is kind of a hack to support
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
-        chownToBuilder(chrootRootDir + "/etc");
+        if (parsedDrv->useUidRange())
+            chownToBuilder(chrootRootDir + "/etc");
 
         if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
             throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name);
@@ -1908,6 +1909,10 @@ void LocalDerivationGoal::runChild()
                 }
             }
 
+            /* Make /etc unwritable */
+            if (!parsedDrv->useUidRange())
+                chmod_(chrootRootDir + "/etc", 0555);
+
             /* Unshare this mount namespace. This is necessary because
                pivot_root() below changes the root of the mount
                namespace. This means that the call to setns() in

src/libstore/builtins/buildenv.cc

@@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                         throw Error(
                                 "files '%1%' and '%2%' have the same priority %3%; "
                                 "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
-                                "or type 'nix profile install --help' if using 'nix profile' to find out how"
+                                "or type 'nix profile install --help' if using 'nix profile' to find out how "
                                 "to change the priority of one of the conflicting packages"
                                 " (0 being the highest priority)",
                                 srcFile, readLink(dstFile), priority);

src/libstore/derived-path.cc

@@ -20,11 +20,12 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
     // Fallback for the input-addressed derivation case: We expect to always be
     // able to print the output paths, so let’s do it
     const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
-    for (const auto& output : outputs) {
+    for (const auto & output : outputs) {
         auto knownOutput = get(knownOutputs, output);
-        res["outputs"][output] = (knownOutput && *knownOutput)
-          ? store->printStorePath(**knownOutput)
-          : nullptr;
+        if (knownOutput && *knownOutput)
+            res["outputs"][output] = store->printStorePath(**knownOutput);
+        else
+            res["outputs"][output] = nullptr;
     }
     return res;
 }

src/libstore/gc.cc

@@ -77,60 +77,73 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
 }
 
 
-void LocalStore::addTempRoot(const StorePath & path)
+void LocalStore::createTempRootsFile()
 {
-    auto state(_state.lock());
+    auto fdTempRoots(_fdTempRoots.lock());
 
     /* Create the temporary roots file for this process. */
-    if (!state->fdTempRoots) {
+    if (*fdTempRoots) return;
 
-        while (1) {
-            if (pathExists(fnTempRoots))
-                /* It *must* be stale, since there can be no two
-                   processes with the same pid. */
-                unlink(fnTempRoots.c_str());
+    while (1) {
+        if (pathExists(fnTempRoots))
+            /* It *must* be stale, since there can be no two
+               processes with the same pid. */
+            unlink(fnTempRoots.c_str());
 
-            state->fdTempRoots = openLockFile(fnTempRoots, true);
+        *fdTempRoots = openLockFile(fnTempRoots, true);
 
-            debug("acquiring write lock on '%s'", fnTempRoots);
-            lockFile(state->fdTempRoots.get(), ltWrite, true);
+        debug("acquiring write lock on '%s'", fnTempRoots);
+        lockFile(fdTempRoots->get(), ltWrite, true);
 
-            /* Check whether the garbage collector didn't get in our
-               way. */
-            struct stat st;
-            if (fstat(state->fdTempRoots.get(), &st) == -1)
-                throw SysError("statting '%1%'", fnTempRoots);
-            if (st.st_size == 0) break;
+        /* Check whether the garbage collector didn't get in our
+           way. */
+        struct stat st;
+        if (fstat(fdTempRoots->get(), &st) == -1)
+            throw SysError("statting '%1%'", fnTempRoots);
+        if (st.st_size == 0) break;
 
-            /* The garbage collector deleted this file before we could
-               get a lock.  (It won't delete the file after we get a
-               lock.)  Try again. */
-        }
+        /* The garbage collector deleted this file before we could get
+           a lock.  (It won't delete the file after we get a lock.)
+           Try again. */
+    }
+}
 
+
+void LocalStore::addTempRoot(const StorePath & path)
+{
+    createTempRootsFile();
+
+    /* Open/create the global GC lock file. */
+    {
+        auto fdGCLock(_fdGCLock.lock());
+        if (!*fdGCLock)
+            *fdGCLock = openGCLock();
     }
 
-    if (!state->fdGCLock)
-        state->fdGCLock = openGCLock();
-
  restart:
-    FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
+    /* Try to acquire a shared global GC lock (non-blocking). This
+       only succeeds if the garbage collector is not currently
+       running. */
+    FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, "");
 
     if (!gcLock.acquired) {
         /* We couldn't get a shared global GC lock, so the garbage
            collector is running. So we have to connect to the garbage
            collector and inform it about our root. */
-        if (!state->fdRootsSocket) {
+        auto fdRootsSocket(_fdRootsSocket.lock());
+
+        if (!*fdRootsSocket) {
             auto socketPath = stateDir.get() + gcSocketPath;
             debug("connecting to '%s'", socketPath);
-            state->fdRootsSocket = createUnixDomainSocket();
+            *fdRootsSocket = createUnixDomainSocket();
             try {
-                nix::connect(state->fdRootsSocket.get(), socketPath);
+                nix::connect(fdRootsSocket->get(), socketPath);
             } catch (SysError & e) {
                 /* The garbage collector may have exited, so we need to
                    restart. */
                 if (e.errNo == ECONNREFUSED) {
                     debug("GC socket connection refused");
-                    state->fdRootsSocket.close();
+                    fdRootsSocket->close();
                     goto restart;
                 }
                 throw;
@@ -139,9 +152,9 @@ void LocalStore::addTempRoot(const StorePath & path)
 
         try {
             debug("sending GC root '%s'", printStorePath(path));
-            writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
+            writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false);
             char c;
-            readFull(state->fdRootsSocket.get(), &c, 1);
+            readFull(fdRootsSocket->get(), &c, 1);
             assert(c == '1');
             debug("got ack for GC root '%s'", printStorePath(path));
         } catch (SysError & e) {
@@ -149,20 +162,21 @@ void LocalStore::addTempRoot(const StorePath & path)
                restart. */
             if (e.errNo == EPIPE || e.errNo == ECONNRESET) {
                 debug("GC socket disconnected");
-                state->fdRootsSocket.close();
+                fdRootsSocket->close();
                 goto restart;
             }
             throw;
         } catch (EndOfFile & e) {
             debug("GC socket disconnected");
-            state->fdRootsSocket.close();
+            fdRootsSocket->close();
             goto restart;
         }
     }
 
-    /* Append the store path to the temporary roots file. */
+    /* Record the store path in the temporary roots file so it will be
+       seen by a future run of the garbage collector. */
     auto s = printStorePath(path) + '\0';
-    writeFull(state->fdTempRoots.get(), s);
+    writeFull(_fdTempRoots.lock()->get(), s);
 }
 
 

src/libstore/globals.hh

@@ -281,7 +281,10 @@ public:
           `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if
           `NIX_REMOTE` is `daemon`). Obviously, this should not be used in
           multi-user settings with untrusted users.
-        )"};
+
+          Defaults to `nixbld` when running as root, *empty* otherwise.
+        )",
+        {}, false};
 
     Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids",
         R"(

src/libstore/http-binary-cache-store.cc

@@ -56,7 +56,7 @@ public:
     void init() override
     {
         // FIXME: do this lazily?
-        if (auto cacheInfo = diskCache->cacheExists(cacheUri)) {
+        if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) {
             wantMassQuery.setDefault(cacheInfo->wantMassQuery);
             priority.setDefault(cacheInfo->priority);
         } else {

src/libstore/local-store.cc

@@ -439,9 +439,9 @@ LocalStore::~LocalStore()
     }
 
     try {
-        auto state(_state.lock());
-        if (state->fdTempRoots) {
-            state->fdTempRoots = -1;
+        auto fdTempRoots(_fdTempRoots.lock());
+        if (*fdTempRoots) {
+            *fdTempRoots = -1;
             unlink(fnTempRoots.c_str());
         }
     } catch (...) {

src/libstore/local-store.hh

@@ -59,15 +59,6 @@ private:
         struct Stmts;
         std::unique_ptr<Stmts> stmts;
 
-        /* The global GC lock  */
-        AutoCloseFD fdGCLock;
-
-        /* The file to which we write our temporary roots. */
-        AutoCloseFD fdTempRoots;
-
-        /* Connection to the garbage collector. */
-        AutoCloseFD fdRootsSocket;
-
         /* The last time we checked whether to do an auto-GC, or an
            auto-GC finished. */
         std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@@ -156,6 +147,21 @@ public:
 
     void addTempRoot(const StorePath & path) override;
 
+private:
+
+    void createTempRootsFile();
+
+    /* The file to which we write our temporary roots. */
+    Sync<AutoCloseFD> _fdTempRoots;
+
+    /* The global GC lock. */
+    Sync<AutoCloseFD> _fdGCLock;
+
+    /* Connection to the garbage collector. */
+    Sync<AutoCloseFD> _fdRootsSocket;
+
+public:
+
     void addIndirectRoot(const Path & path) override;
 
 private:

src/libstore/lock.cc

@@ -123,8 +123,12 @@ struct AutoUserLock : UserLock
 
     std::vector<gid_t> getSupplementaryGIDs() override { return {}; }
 
-    static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useChroot)
+    static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useUserNamespace)
     {
+        #if !defined(__linux__)
+        useUserNamespace = false;
+        #endif
+
         settings.requireExperimentalFeature(Xp::AutoAllocateUids);
         assert(settings.startId > 0);
         assert(settings.uidCount % maxIdsPerBuild == 0);
@@ -157,7 +161,7 @@ struct AutoUserLock : UserLock
                 auto lock = std::make_unique<AutoUserLock>();
                 lock->fdUserLock = std::move(fd);
                 lock->firstUid = firstUid;
-                if (useChroot)
+                if (useUserNamespace)
                     lock->firstGid = firstUid;
                 else {
                     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@@ -174,10 +178,10 @@ struct AutoUserLock : UserLock
     }
 };
 
-std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
+std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace)
 {
     if (settings.autoAllocateUids)
-        return AutoUserLock::acquire(nrIds, useChroot);
+        return AutoUserLock::acquire(nrIds, useUserNamespace);
     else
         return SimpleUserLock::acquire();
 }
@@ -185,7 +189,7 @@ std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot)
 bool useBuildUsers()
 {
     #if __linux__
-    static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0;
+    static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0;
     return b;
     #elif __APPLE__
     static bool b = settings.buildUsersGroup != "" && getuid() == 0;

src/libstore/lock.hh

@@ -31,7 +31,7 @@ struct UserLock
 
 /* Acquire a user lock for a UID range of size `nrIds`. Note that this
    may return nullptr if no user is available. */
-std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot);
+std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace);
 
 bool useBuildUsers();
 

src/libstore/nar-info-disk-cache.cc

@@ -84,11 +84,10 @@ public:
 
     Sync<State> _state;
 
-    NarInfoDiskCacheImpl()
+    NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite")
     {
         auto state(_state.lock());
 
-        Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite";
         createDirs(dirOf(dbPath));
 
         state->db = SQLite(dbPath);
@@ -98,7 +97,7 @@ public:
         state->db.exec(schema);
 
         state->insertCache.create(state->db,
-            "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
+            "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;");
 
         state->queryCache.create(state->db,
             "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?");
@@ -166,20 +165,61 @@ public:
         return i->second;
     }
 
-    void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
+private:
+
+    std::optional<Cache> queryCacheRaw(State & state, const std::string & uri)
     {
-        retrySQLite<void>([&]() {
+        auto i = state.caches.find(uri);
+        if (i == state.caches.end()) {
+            auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl));
+            if (!queryCache.next())
+                return std::nullopt;
+            auto cache = Cache {
+                .id = (int) queryCache.getInt(0),
+                .storeDir = queryCache.getStr(1),
+                .wantMassQuery = queryCache.getInt(2) != 0,
+                .priority = (int) queryCache.getInt(3),
+            };
+            state.caches.emplace(uri, cache);
+        }
+        return getCache(state, uri);
+    }
+
+public:
+    int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override
+    {
+        return retrySQLite<int>([&]() {
             auto state(_state.lock());
+            SQLiteTxn txn(state->db);
 
-            // FIXME: race
+            // To avoid the race, we have to check if maybe someone hasn't yet created
+            // the cache for this URI in the meantime.
+            auto cache(queryCacheRaw(*state, uri));
 
-            state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec();
-            assert(sqlite3_changes(state->db) == 1);
-            state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority};
+            if (cache)
+                return cache->id;
+
+            Cache ret {
+                .id = -1, // set below
+                .storeDir = storeDir,
+                .wantMassQuery = wantMassQuery,
+                .priority = priority,
+            };
+
+            {
+                auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority));
+                assert(r.next());
+                ret.id = (int) r.getInt(0);
+            }
+
+            state->caches[uri] = ret;
+
+            txn.commit();
+            return ret.id;
         });
     }
 
-    std::optional<CacheInfo> cacheExists(const std::string & uri) override
+    std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override
     {
         return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> {
             auto state(_state.lock());
@@ -196,6 +236,7 @@ public:
             auto & cache(getCache(*state, uri));
 
             return CacheInfo {
+                .id = cache.id,
                 .wantMassQuery = cache.wantMassQuery,
                 .priority = cache.priority
             };
@@ -359,4 +400,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache()
     return cache;
 }
 
+ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath)
+{
+    return make_ref<NarInfoDiskCacheImpl>(dbPath);
+}
+
 }

src/libstore/nar-info-disk-cache.hh

@@ -13,16 +13,17 @@ public:
 
     virtual ~NarInfoDiskCache() { }
 
-    virtual void createCache(const std::string & uri, const Path & storeDir,
+    virtual int createCache(const std::string & uri, const Path & storeDir,
         bool wantMassQuery, int priority) = 0;
 
     struct CacheInfo
     {
+        int id;
         bool wantMassQuery;
         int priority;
     };
 
-    virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0;
+    virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0;
 
     virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
         const std::string & uri, const std::string & hashPart) = 0;
@@ -45,4 +46,6 @@ public:
    multiple threads. */
 ref<NarInfoDiskCache> getNarInfoDiskCache();
 
+ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath);
+
 }

src/libstore/realisation.hh

@@ -93,4 +93,14 @@ struct RealisedPath {
     GENERATE_CMP(RealisedPath, me->raw);
 };
 
+class MissingRealisation : public Error
+{
+public:
+    MissingRealisation(DrvOutput & outputId)
+        : Error( "cannot operate on an output of the "
+                "unbuilt derivation '%s'",
+                outputId.to_string())
+    {}
+};
+
 }

src/libstore/remote-store.cc

@@ -879,10 +879,7 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults(
                                 auto realisation =
                                     queryRealisation(outputId);
                                 if (!realisation)
-                                    throw Error(
-                                        "cannot operate on an output of unbuilt "
-                                        "content-addressed derivation '%s'",
-                                        outputId.to_string());
+                                    throw MissingRealisation(outputId);
                                 res.builtOutputs.emplace(realisation->id, *realisation);
                             } else {
                                 // If ca-derivations isn't enabled, assume that

src/libstore/s3-binary-cache-store.cc

@@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual
 
     void init() override
     {
-        if (auto cacheInfo = diskCache->cacheExists(getUri())) {
+        if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) {
             wantMassQuery.setDefault(cacheInfo->wantMassQuery);
             priority.setDefault(cacheInfo->priority);
         } else {

src/libstore/sqlite.cc

@@ -36,6 +36,15 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor
         throw SQLiteError(path, err, exterr, std::move(hf));
 }
 
+static void traceSQL(void * x, const char * sql)
+{
+    // wacky delimiters:
+    //   so that we're quite unambiguous without escaping anything
+    // notice instead of trace:
+    //   so that this can be enabled without getting the firehose in our face.
+    notice("SQL<[%1%]>", sql);
+};
+
 SQLite::SQLite(const Path & path, bool create)
 {
     // useSQLiteWAL also indicates what virtual file system we need.  Using
@@ -49,6 +58,11 @@ SQLite::SQLite(const Path & path, bool create)
     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
         SQLiteError::throw_(db, "setting timeout");
 
+    if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") {
+        // To debug sqlite statements; trace all of them
+        sqlite3_trace(db, &traceSQL, nullptr);
+    }
+
     exec("pragma foreign_keys = 1");
 }
 

src/libstore/tests/nar-info-disk-cache.cc

@@ -0,0 +1,122 @@
+#include "nar-info-disk-cache.hh"
+
+#include <gtest/gtest.h>
+#include "sqlite.hh"
+#include <sqlite3.h>
+
+
+namespace nix {
+
+TEST(NarInfoDiskCacheImpl, create_and_read) {
+    // This is a large single test to avoid some setup overhead.
+
+    int prio = 12345;
+    bool wantMassQuery = true;
+
+    Path tmpDir = createTempDir();
+    AutoDelete delTmpDir(tmpDir);
+    Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite");
+
+    int savedId;
+    int barId;
+    SQLite db;
+    SQLiteStmt getIds;
+
+    {
+        auto cache = getTestNarInfoDiskCache(dbPath);
+
+        // Set up "background noise" and check that different caches receive different ids
+        {
+            auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio);
+            auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12);
+            ASSERT_NE(bc1, bc2);
+            barId = bc1;
+        }
+
+        // Check that the fields are saved and returned correctly. This does not test
+        // the select statement yet, because of in-memory caching.
+        savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);;
+        {
+            auto r = cache->upToDateCacheExists("http://foo");
+            ASSERT_TRUE(r);
+            ASSERT_EQ(r->priority, prio);
+            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(savedId, r->id);
+        }
+
+        // We're going to pay special attention to the id field because we had a bug
+        // that changed it.
+        db = SQLite(dbPath);
+        getIds.create(db, "select id from BinaryCaches where url = 'http://foo'");
+
+        {
+            auto q(getIds.use());
+            ASSERT_TRUE(q.next());
+            ASSERT_EQ(savedId, q.getInt(0));
+            ASSERT_FALSE(q.next());
+        }
+
+        // Pretend that the caches are older, but keep one up to date, as "background noise"
+        db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';");
+
+        // This shows that the in-memory cache works
+        {
+            auto r = cache->upToDateCacheExists("http://foo");
+            ASSERT_TRUE(r);
+            ASSERT_EQ(r->priority, prio);
+            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+        }
+    }
+
+    {
+        // We can't clear the in-memory cache, so we use a new cache object. This is
+        // more realistic anyway.
+        auto cache2 = getTestNarInfoDiskCache(dbPath);
+
+        {
+            auto r = cache2->upToDateCacheExists("http://foo");
+            ASSERT_FALSE(r);
+        }
+
+        // "Update", same data, check that the id number is reused
+        cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);
+
+        {
+            auto r = cache2->upToDateCacheExists("http://foo");
+            ASSERT_TRUE(r);
+            ASSERT_EQ(r->priority, prio);
+            ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+            ASSERT_EQ(r->id, savedId);
+        }
+
+        {
+            auto q(getIds.use());
+            ASSERT_TRUE(q.next());
+            auto currentId = q.getInt(0);
+            ASSERT_FALSE(q.next());
+            ASSERT_EQ(currentId, savedId);
+        }
+
+        // Check that the fields can be modified, and the id remains the same
+        {
+            auto r0 = cache2->upToDateCacheExists("https://bar");
+            ASSERT_FALSE(r0);
+
+            cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10);
+            auto r = cache2->upToDateCacheExists("https://bar");
+            ASSERT_EQ(r->wantMassQuery, !wantMassQuery);
+            ASSERT_EQ(r->priority, prio + 10);
+            ASSERT_EQ(r->id, barId);
+        }
+
+        // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp)
+        // {
+        //     cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20);
+        //     auto r = cache2->upToDateCacheExists("https://bar");
+        //     ASSERT_EQ(r->wantMassQuery, wantMassQuery);
+        //     ASSERT_EQ(r->priority, prio + 20);
+        // }
+    }
+}
+
+}

src/nix/why-depends.cc

@@ -95,23 +95,13 @@ struct CmdWhyDepends : SourceExprCommand
          * to build.
          */
         auto dependency = parseInstallable(store, _dependency);
-        auto derivedDependency = dependency->toDerivedPath();
-        auto optDependencyPath = std::visit(overloaded {
-            [](const DerivedPath::Opaque & nodrv) -> std::optional<StorePath> {
-                return { nodrv.path };
-            },
-            [&](const DerivedPath::Built & hasdrv) -> std::optional<StorePath> {
-                if (hasdrv.outputs.size() != 1) {
-                    throw Error("argument '%s' should evaluate to one store path", dependency->what());
-                }
-                auto outputMap = store->queryPartialDerivationOutputMap(hasdrv.drvPath);
-                auto maybePath = outputMap.find(*hasdrv.outputs.begin());
-                if (maybePath == outputMap.end()) {
-                    throw Error("unexpected end of iterator");
-                }
-                return maybePath->second;
-            },
-        }, derivedDependency.raw());
+        auto optDependencyPath = [&]() -> std::optional<StorePath> {
+            try {
+                return {Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency)};
+            } catch (MissingRealisation &) {
+                return std::nullopt;
+            }
+        }();
 
         StorePathSet closure;
         store->computeFSClosure({packagePath}, closure, false, false);

tests/impure-derivations.sh

@@ -12,6 +12,7 @@ clearStore
 # Basic test of impure derivations: building one a second time should not use the previous result.
 printf 0 > $TEST_ROOT/counter
 
+nix build --dry-run --json --file ./impure-derivations.nix impure.all
 json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
 path1=$(echo $json | jq -r .[].outputs.out)
 path1_stuff=$(echo $json | jq -r .[].outputs.stuff)

tests/linux-sandbox.sh

@@ -37,3 +37,6 @@ nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
 (! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
 if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi
 grep -q 'may not be deterministic' $TEST_ROOT/log
+
+# Test that sandboxed builds cannot write to /etc easily
+(! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store)

tests/local.mk

@@ -90,6 +90,7 @@ nix_tests = \
   fmt.sh \
   eval-store.sh \
   why-depends.sh \
+  ca/why-depends.sh \
   import-derivation.sh \
   ca/import-derivation.sh \
   nix_path.sh \

tests/systemd-nspawn.nix

@@ -56,12 +56,12 @@ runCommand "test"
     # Make /run a tmpfs to shut up a systemd warning.
     mkdir /run
     mount -t tmpfs none /run
-    chmod 0700 /run
 
     mount -t cgroup2 none /sys/fs/cgroup
 
     mkdir -p $out
 
+    chmod +w /etc
     touch /etc/os-release
     echo a5ea3f98dedc0278b6f3cc8c37eeaeac > /etc/machine-id
 

tests/why-depends.sh

@@ -6,6 +6,9 @@ cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME
 
 cd $TEST_HOME
 
+nix why-depends --derivation --file ./dependencies.nix input2_drv input1_drv
+nix why-depends --file ./dependencies.nix input2_drv input1_drv
+
 nix-build ./dependencies.nix -A input0_drv -o dep
 nix-build ./dependencies.nix -o toplevel