Merge pull request #8350 from NixOS/backport-8154-to-2.15-maintenance

[Backport 2.15-maintenance] Make `nix-collect-garbage -d` look into more places

.version

@@ -1 +1 @@
-2.15.0
+2.15.1

doc/manual/src/SUMMARY.md.in

@@ -98,7 +98,6 @@
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
   - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)

flake.nix

@@ -11,7 +11,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -320,18 +320,12 @@
           };
           let
             canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-
-            sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith {
-              filter = (path: type:
-                let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path);
-                in ! lib.any (re: builtins.match re relPath != null) rxs);
-              src = origSrc;
-            };
           in currentStdenv.mkDerivation (finalAttrs: {
             name = "nix-${version}";
             inherit version;
 
-            src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self;
+            src = self;
+
             VERSION_SUFFIX = versionSuffix;
 
             outputs = [ "out" "dev" "doc" ];

src/libcmd/installable-flake.cc

@@ -235,7 +235,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
         }
     }
 
-    return InstallableValue::nixpkgsFlakeRef();
+    return defaultNixpkgsFlakeRef();
 }
 
 }

src/libcmd/installable-flake.hh

@@ -67,9 +67,22 @@ struct InstallableFlake : InstallableValue
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
-    FlakeRef nixpkgsFlakeRef() const override;
+    FlakeRef nixpkgsFlakeRef() const;
 };
 
+/**
+ * Default flake ref for referring to Nixpkgs. For flakes that don't
+ * have their own Nixpkgs input, or other installables.
+ *
+ * It is a layer violation for Nix to know about Nixpkgs; currently just
+ * `nix develop` does. Be wary of using this /
+ * `InstallableFlake::nixpkgsFlakeRef` more places.
+ */
+static inline FlakeRef defaultNixpkgsFlakeRef()
+{
+    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+}
+
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake);

src/libcmd/installable-value.hh

@@ -96,11 +96,6 @@ struct InstallableValue : Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual FlakeRef nixpkgsFlakeRef() const
-    {
-        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
-    }
-
     static InstallableValue & require(Installable & installable);
     static ref<InstallableValue> require(ref<Installable> installable);
 };

src/libstore/derivations.cc

@@ -980,7 +980,8 @@ nlohmann::json DerivationOutput::toJSON(
 
 DerivationOutput DerivationOutput::fromJSON(
     const Store & store, std::string_view drvName, std::string_view outputName,
-    const nlohmann::json & _json)
+    const nlohmann::json & _json,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     std::set<std::string_view> keys;
     auto json = (std::map<std::string, nlohmann::json>) _json;
@@ -1019,6 +1020,7 @@ DerivationOutput DerivationOutput::fromJSON(
     }
 
     else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
+        xpSettings.require(Xp::CaDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::CAFloating {
             .method = method,
@@ -1031,6 +1033,7 @@ DerivationOutput DerivationOutput::fromJSON(
     }
 
     else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
+        xpSettings.require(Xp::ImpureDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::Impure {
             .method = method,

src/libstore/derivations.hh

@@ -136,11 +136,15 @@ struct DerivationOutput : _DerivationOutputRaw
         const Store & store,
         std::string_view drvName,
         std::string_view outputName) const;
+    /**
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
     static DerivationOutput fromJSON(
         const Store & store,
         std::string_view drvName,
         std::string_view outputName,
-        const nlohmann::json & json);
+        const nlohmann::json & json,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
 
 typedef std::map<std::string, DerivationOutput> DerivationOutputs;

src/libstore/derived-path.cc

@@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
 std::string DerivedPath::Built::to_string(const Store & store) const
 {
     return store.printStorePath(drvPath)
-        + "!"
+        + '^'
+        + outputs.to_string();
+}
+
+std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+{
+    return store.printStorePath(drvPath)
+        + '!'
         + outputs.to_string();
 }
 
 std::string DerivedPath::to_string(const Store & store) const
 {
-    return std::visit(
-        [&](const auto & req) { return req.to_string(store); },
-        this->raw());
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & req) { return req.to_string(store); },
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
+}
+
+std::string DerivedPath::to_string_legacy(const Store & store) const
+{
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
 }
 
 
@@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
     };
 }
 
-DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
 {
-    size_t n = s.find("!");
+    size_t n = s.find(separator);
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
         : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
 }
 
+DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "^");
+}
+
+DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "!");
+}
+
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
 {
     RealisedPath::Set res;

src/libstore/derived-path.hh

@@ -48,8 +48,18 @@ struct DerivedPathBuilt {
     StorePath drvPath;
     OutputsSpec outputs;
 
+    /**
+     * Uses `^` as the separator
+     */
     std::string to_string(const Store & store) const;
-    static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * The caller splits on the separator, so it works for both variants.
+     */
+    static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
     nlohmann::json toJSON(ref<Store> store) const;
 
     GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
@@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw {
         return static_cast<const Raw &>(*this);
     }
 
+    /**
+     * Uses `^` as the separator
+     */
     std::string to_string(const Store & store) const;
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * Uses `^` as the separator
+     */
     static DerivedPath parse(const Store & store, std::string_view);
+    /**
+     * Uses `!` as the separator
+     */
+    static DerivedPath parseLegacy(const Store & store, std::string_view);
 };
 
 /**

src/libstore/remote-store.cc

@@ -90,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
 DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
 {
     auto s = readString(from);
-    return DerivedPath::parse(store, s);
+    return DerivedPath::parseLegacy(store, s);
 }
 
 void write(const Store & store, Sink & out, const DerivedPath & req)
 {
-    out << req.to_string(store);
+    out << req.to_string_legacy(store);
 }
 
 

src/libstore/tests/derivation.cc

@@ -1,6 +1,7 @@
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
+#include "experimental-features.hh"
 #include "derivations.hh"
 
 #include "tests/libstore.hh"
@@ -9,10 +10,32 @@ namespace nix {
 
 class DerivationTest : public LibStoreTest
 {
+public:
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
 };
 
-#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
-    TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) {    \
+class CaDerivationTest : public DerivationTest
+{
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "ca-derivations");
+    }
+};
+
+class ImpureDerivationTest : public DerivationTest
+{
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "impure-derivations");
+    }
+};
+
+#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
+    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {    \
         using nlohmann::literals::operator "" _json;           \
         ASSERT_EQ(                                             \
             STR ## _json,                                      \
@@ -22,7 +45,7 @@ class DerivationTest : public LibStoreTest
                 OUTPUT_NAME));                                 \
     }                                                          \
                                                                \
-    TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) {  \
+    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) {  \
         using nlohmann::literals::operator "" _json;           \
         ASSERT_EQ(                                             \
             DerivationOutput { VAL },                          \
@@ -30,10 +53,11 @@ class DerivationTest : public LibStoreTest
                 *store,                                        \
                 DRV_NAME,                                      \
                 OUTPUT_NAME,                                   \
-                STR ## _json));                                \
+                STR ## _json,                                  \
+                mockXpSettings));                              \
     }
 
-TEST_JSON(inputAddressed,
+TEST_JSON(DerivationTest, inputAddressed,
     R"({
         "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
     })",
@@ -42,7 +66,7 @@ TEST_JSON(inputAddressed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(caFixed,
+TEST_JSON(DerivationTest, caFixed,
     R"({
         "hashAlgo": "r:sha256",
         "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
@@ -56,7 +80,7 @@ TEST_JSON(caFixed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(caFloating,
+TEST_JSON(CaDerivationTest, caFloating,
     R"({
         "hashAlgo": "r:sha256"
     })",
@@ -66,12 +90,12 @@ TEST_JSON(caFloating,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(deferred,
+TEST_JSON(DerivationTest, deferred,
     R"({ })",
     DerivationOutput::Deferred { },
     "drv-name", "output-name")
 
-TEST_JSON(impure,
+TEST_JSON(ImpureDerivationTest, impure,
     R"({
         "hashAlgo": "r:sha256",
         "impure": true

src/libstore/tests/derived-path.cc

@@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init)
 {
 }
 
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathTest,
+    prop_legacy_round_rip,
+    (const DerivedPath & o))
+{
+    RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store)));
+}
+
 RC_GTEST_FIXTURE_PROP(
     DerivedPathTest,
     prop_round_rip,

src/libutil/config.cc

@@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings()
         set(s.first, s.second);
 }
 
-// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features.
-static bool applicable(const Config::SettingData & sd)
-{
-    return !sd.isAlias
-        && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature);
-}
-
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (auto & opt : _settings)
-        if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden))
+        if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
@@ -154,7 +147,7 @@ nlohmann::json Config::toJSON()
 {
     auto res = nlohmann::json::object();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (!s.second.isAlias)
             res.emplace(s.first, s.second.setting->toJSON());
     return res;
 }
@@ -163,7 +156,7 @@ std::string Config::toKeyValue()
 {
     auto res = std::string();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (s.second.isAlias)
             res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
     return res;
 }
@@ -171,9 +164,6 @@ std::string Config::toKeyValue()
 void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings) {
-        /* We do include args for settings gated on disabled
-           experimental-features. The args themselves however will also be
-           gated on any experimental feature the underlying setting is. */
         if (!s.second.isAlias)
             s.second.setting->convertToArg(args, category);
     }

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -77,7 +77,12 @@ static int main_nix_collect_garbage(int argc, char * * argv)
             return true;
         });
 
-        if (removeOld) removeOldGenerations(profilesDir());
+        if (removeOld) {
+            std::set<Path> dirsToClean = {
+                profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())};
+            for (auto & dir : dirsToClean)
+                removeOldGenerations(dir);
+        }
 
         // Run the actual garbage collector.
         if (!dryRun) {

src/nix/develop.cc

@@ -252,7 +252,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     throw Error("get-env.sh failed to produce an environment");
 }
 
-struct Common : InstallableValueCommand, MixProfile
+struct Common : InstallableCommand, MixProfile
 {
     std::set<std::string> ignoreVars{
         "BASHOPTS",
@@ -374,7 +374,7 @@ struct Common : InstallableValueCommand, MixProfile
         return res;
     }
 
-    StorePath getShellOutPath(ref<Store> store, ref<InstallableValue> installable)
+    StorePath getShellOutPath(ref<Store> store, ref<Installable> installable)
     {
         auto path = installable->getStorePath();
         if (path && hasSuffix(path->to_string(), "-env"))
@@ -393,7 +393,7 @@ struct Common : InstallableValueCommand, MixProfile
     }
 
     std::pair<BuildEnvironment, std::string>
-    getBuildEnvironment(ref<Store> store, ref<InstallableValue> installable)
+    getBuildEnvironment(ref<Store> store, ref<Installable> installable)
     {
         auto shellOutPath = getShellOutPath(store, installable);
 
@@ -481,7 +481,7 @@ struct CmdDevelop : Common, MixEnvironment
           ;
     }
 
-    void run(ref<Store> store, ref<InstallableValue> installable) override
+    void run(ref<Store> store, ref<Installable> installable) override
     {
         auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable);
 
@@ -538,10 +538,14 @@ struct CmdDevelop : Common, MixEnvironment
             nixpkgsLockFlags.inputOverrides = {};
             nixpkgsLockFlags.inputUpdates = {};
 
+            auto nixpkgs = defaultNixpkgsFlakeRef();
+            if (auto * i = dynamic_cast<const InstallableFlake *>(&*installable))
+                nixpkgs = i->nixpkgsFlakeRef();
+
             auto bashInstallable = make_ref<InstallableFlake>(
                 this,
                 state,
-                installable->nixpkgsFlakeRef(),
+                std::move(nixpkgs),
                 "bashInteractive",
                 DefaultOutputs(),
                 Strings{},
@@ -605,7 +609,7 @@ struct CmdPrintDevEnv : Common, MixJSON
 
     Category category() override { return catUtility; }
 
-    void run(ref<Store> store, ref<InstallableValue> installable) override
+    void run(ref<Store> store, ref<Installable> installable) override
     {
         auto buildEnvironment = getBuildEnvironment(store, installable).first;
 

tests/ca/derivation-json.sh

@@ -16,6 +16,9 @@ drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json)
 # With --dry-run nothing is actually written
 [[ ! -e "$drvPath3" ]]
 
+# But the JSON is rejected without the experimental feature
+expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled"
+
 # Without --dry-run it is actually written
 drvPath4=$(nix derivation add < $TEST_HOME/foo.json)
 [[ "$drvPath4" = "$drvPath3" ]]

tests/experimental-features.sh

@@ -1,25 +1,27 @@
 source common.sh
 
-# Without flakes, flake options should not show up
-# With flakes, flake options should show up
-
-function both_ways {
-    nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake
-    nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake
-
-    # Also, the order should not matter
-    nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake
-    nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake
-}
-
-# Simple case, the configuration effects the running command
-both_ways show-config
-
-# Skipping for now, because we actually *do* want these to show up in
-# the manual, just be marked experimental. Will reenable once the manual
-# generation takes advantage of the JSON metadata on this.
-
-# both_ways store gc --help
+# Skipping these two for now, because we actually *do* want flags and
+# config settings to always show up in the manual, just be marked
+# experimental. Will reenable once the manual generation takes advantage
+# of the JSON metadata on this.
+#
+# # Without flakes, flake options should not show up
+# # With flakes, flake options should show up
+#
+# function grep_both_ways {
+#     nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake
+#     nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake
+#
+#     # Also, the order should not matter
+#     nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake
+#     nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake
+# }
+#
+# # Simple case, the configuration effects the running command
+# grep_both_ways show-config
+#
+# # Medium case, the configuration effects --help
+# grep_both_ways store gc --help
 
 expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no'
 nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no'

tests/gc.sh

@@ -52,9 +52,7 @@ rmdir $NIX_STORE_DIR/.links
 rmdir $NIX_STORE_DIR
 
 ## Test `nix-collect-garbage -d`
-# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
-# using them
-if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+testCollectGarbageD () {
     clearProfiles
     # Run two `nix-env` commands, should create two generations of
     # the profile
@@ -66,4 +64,17 @@ if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
     # left
     nix-collect-garbage -d
     [[ $(nix-env --list-generations | wc -l) -eq 1 ]]
+}
+# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
+# using them
+if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+    testCollectGarbageD
+
+    # Run the same test, but forcing the profiles at their legacy location under
+    # /nix/var/nix.
+    #
+    # Regression test for #8294
+    rm ~/.nix-profile
+    ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile
+    testCollectGarbageD
 fi

tests/impure-derivations.sh

@@ -10,6 +10,15 @@ clearStore
 # Basic test of impure derivations: building one a second time should not use the previous result.
 printf 0 > $TEST_ROOT/counter
 
+# `nix derivation add` with impure derivations work
+drvPath=$(nix-instantiate ./impure-derivations.nix -A impure)
+nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json
+drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json)
+[[ "$drvPath" = "$drvPath2" ]]
+
+# But only with the experimental feature!
+expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled"
+
 nix build --dry-run --json --file ./impure-derivations.nix impure.all
 json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
 path1=$(echo $json | jq -r .[].outputs.out)

tests/nix-shell.sh

@@ -98,6 +98,18 @@ nix develop -f "$shellDotNix" shellDrv -c echo foo |& grepQuiet foo
 nix print-dev-env -f "$shellDotNix" shellDrv > $TEST_ROOT/dev-env.sh
 nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json
 
+# Test with raw drv
+
+shellDrv=$(nix-instantiate "$shellDotNix" -A shellDrv.out)
+
+nix develop $shellDrv -c bash -c '[[ -n $stdenv ]]'
+
+nix print-dev-env $shellDrv > $TEST_ROOT/dev-env2.sh
+nix print-dev-env $shellDrv --json > $TEST_ROOT/dev-env2.json
+
+diff $TEST_ROOT/dev-env{,2}.sh
+diff $TEST_ROOT/dev-env{,2}.json
+
 # Ensure `nix print-dev-env --json` contains variable assignments.
 [[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]]