Ignore delete failures during garbage collection

When running nix as an unprivileged user it may not be able to write to
all paths in the nix store. Ignore deletion failures to fix tests that
run `nix-collect-garbage` in this configuration.

Co-Authored-By: John Ericson <John.Ericson@Obsidian.Systems>

src/libstore/gc.cc

@@ -651,7 +651,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
 
     /* Helper function that deletes a path from the store and throws
        GCLimitReached if we've deleted enough garbage. */
-    auto deleteFromStore = [&](std::string_view baseName) {
+    auto deleteFromStore = [&](std::string_view baseName, bool isKnownPath) {
         Path path = storeDir + "/" + std::string(baseName);
         Path realPath = config->realStoreDir + "/" + std::string(baseName);
 
@@ -671,7 +671,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         results.paths.insert(path);
 
         uint64_t bytesFreed;
-        deleteStorePath(realPath, bytesFreed);
+        deleteStorePath(realPath, bytesFreed, isKnownPath);
 
         results.bytesFreed += bytesFreed;
 
@@ -793,7 +793,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             if (shouldDelete) {
                 try {
                     invalidatePathChecked(path);
-                    deleteFromStore(path.to_string());
+                    deleteFromStore(path.to_string(), true);
                     referrersCache.erase(path);
                 } catch (PathInUse & e) {
                     // If we end up here, it's likely a new occurrence
@@ -844,7 +844,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
                     deleteReferrersClosure(*storePath);
                 else
-                    deleteFromStore(name);
+                    deleteFromStore(name, false);
             }
         } catch (GCLimitReached & e) {
         }

src/libstore/include/nix/store/local-overlay-store.hh

@@ -184,7 +184,7 @@ private:
      * Check which layers the store object exists in to try to avoid
      * needing to remount.
      */
-    void deleteStorePath(const Path & path, uint64_t & bytesFreed) override;
+    void deleteStorePath(const Path & path, uint64_t & bytesFreed, bool isKnownPath) override;
 
     /**
      * Deduplicate by removing store objects from the upper layer that

src/libstore/include/nix/store/local-store.hh

@@ -117,6 +117,20 @@ public:
           > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
         )"};
 
+    Setting<bool> ignoreGcDeleteFailure{
+        this,
+        false,
+        "ignore-gc-delete-failure",
+        R"(
+          Whether to ignore failures when deleting items with the garbage collector.
+
+          Normally the garbage collector will fail with an error if the nix daemon cannot delete a file, with this setting such errors will only be printed as warnings.
+        )",
+        {},
+        true,
+        Xp::LocalOverlayStore,
+    };
+
     static const std::string name()
     {
         return "Local Store";
@@ -312,8 +326,11 @@ public:
      * Called by `collectGarbage` to recursively delete a path.
      * The default implementation simply calls `deletePath`, but it can be
      * overridden by stores that wish to provide their own deletion behaviour.
+     *
+     * @param isKnownPath true if this is a known store path, false if it's
+     *        garbage/unknown content found in the store directory
      */
-    virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed);
+    virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed, bool isKnownPath);
 
     /**
      * Optimise the disk space usage of the Nix store by hard-linking

src/libstore/local-overlay-store.cc

@@ -204,7 +204,7 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re
     remountIfNecessary();
 }
 
-void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
+void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed, bool isKnownPath)
 {
     auto mergedDir = config->realStoreDir.get() + "/";
     if (path.substr(0, mergedDir.length()) != mergedDir) {
@@ -226,7 +226,7 @@ void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed
         } else {
             // Path does not exist in lower store.
             // So we can delete via overlayfs and not need to remount.
-            LocalStore::deleteStorePath(path, bytesFreed);
+            LocalStore::deleteStorePath(path, bytesFreed, isKnownPath);
         }
     }
 }
@@ -246,7 +246,7 @@ void LocalOverlayStore::optimiseStore()
         if (lowerStore->isValidPath(path)) {
             uint64_t bytesFreed = 0;
             // Deduplicate store path
-            deleteStorePath(toRealPath(path), bytesFreed);
+            deleteStorePath(toRealPath(path), bytesFreed, true);
         }
         done++;
         act.progress(done, paths.size());

src/libstore/local-store.cc

@@ -408,9 +408,26 @@ AutoCloseFD LocalStore::openGCLock()
     return toDescriptor(fdGCLock);
 }
 
-void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
+void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed, bool isKnownPath)
 {
-    deletePath(path, bytesFreed);
+    try {
+        deletePath(path, bytesFreed);
+    } catch (SystemError & e) {
+        if (config->ignoreGcDeleteFailure) {
+            logWarning(
+                {.msg = HintFmt(
+                     isKnownPath ? "ignoring failure to remove store path '%1%': %2%"
+                                 : "ignoring failure to remove garbage in store directory '%1%': %2%",
+                     path,
+                     e.info().msg)});
+        } else {
+            e.addTrace(
+                {},
+                isKnownPath ? "While deleting store path '%1%'" : "While deleting garbage in store directory '%1%'",
+                path);
+            throw;
+        }
+    }
 }
 
 LocalStore::~LocalStore()