Add docs

(cherry picked from commit cab03fb779)

.github/workflows/ci.yml

@@ -20,6 +20,9 @@ jobs:
       with:
         fetch-depth: 0
     - uses: cachix/install-nix-action@v20
+      with:
+        # The sandbox would otherwise be disabled by default on Darwin
+        extra_nix_config: "sandbox = true"
     - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
     - uses: cachix/cachix-action@v12
       if: needs.check_secrets.outputs.cachix == 'true'

.version

@@ -1 +1 @@
-2.15.0
+2.15.2

doc/manual/src/SUMMARY.md.in

@@ -92,13 +92,14 @@
   - [Files](command-ref/files.md)
     - [nix.conf](command-ref/conf-file.md)
 - [Architecture](architecture/architecture.md)
+- [Protocols](protocols/protocols.md)
+  - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
 - [Glossary](glossary.md)
 - [Contributing](contributing/contributing.md)
   - [Hacking](contributing/hacking.md)
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
 - [Release Notes](release-notes/release-notes.md)
-  - [Release X.Y (202?-??-??)](release-notes/rl-next.md)
   - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
   - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)
   - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md)

doc/manual/src/protocols/protocols.md

@@ -0,0 +1,4 @@
+# Protocols
+
+This chapter documents various developer-facing interfaces provided by
+Nix.

doc/manual/src/protocols/tarball-fetcher.md

@@ -0,0 +1,40 @@
+# Serving Tarball Flakes
+
+Tarball flakes are served as regular tarballs via HTTP or the file
+system (for `file://` URLs).
+
+An HTTP server can return an "immutable" flakeref appropriate for lock
+files. This allows users to specify a tarball flake input in
+`flake.nix` that requests the latest version of a flake
+(e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock`
+will record a URL whose contents will not change
+(e.g. `https://example.org/hello/<revision>.tar.gz`). To do so, the
+server must return a `Link` header with the `rel` attribute set to
+`immutable`, as follows:
+
+```
+Link: <flakeref>; rel="immutable"
+```
+
+(Note the required `<` and `>` characters around *flakeref*.)
+
+*flakeref* must be a tarball flakeref. It can contain flake attributes
+such as `narHash`, `rev` and `revCount`. If `narHash` is included, its
+value must be the NAR hash of the unpacked tarball (as computed via
+`nix hash path`). Nix checks the contents of the returned tarball
+against the `narHash` attribute. The `rev` and `revCount` attributes
+are useful when the tarball flake is a mirror of a fetcher type that
+has those attributes, such as Git or GitHub. They are not checked by
+Nix.
+
+```
+Link: <https://example.org/hello/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz
+  ?rev=442793d9ec0584f6a6e82fa253850c8085bb150a
+  &revCount=835
+  &narHash=sha256-GUm8Uh/U74zFCwkvt9Mri4DSM%2BmHj3tYhXUkYpiv31M%3D>; rel="immutable"
+```
+
+(The linebreaks in this example are for clarity and must not be included in the actual response.)
+
+For tarball flakes, the value of the `lastModified` flake attribute is
+defined as the timestamp of the newest file inside the tarball.

doc/manual/src/release-notes/rl-2.13.md

@@ -42,3 +42,12 @@
 * You can now disable the global flake registry by setting the `flake-registry`
   configuration option to an empty string. The same can be achieved at runtime with
   `--flake-registry ""`.
+
+* Since 2.13.5, a new function `builtins.readFileType` is available. It is similar to
+  `builtins.readDir` but acts on a single file or directory.
+
+* Since 2.13.5, the `builtins.readDir` function has been optimized when encountering not-yet-known
+  file types from POSIX's `readdir`. In such cases the type of each file was
+  discovered by making multiple syscalls. This change makes these operations
+  lazy such that these lookups will only be performed if the attribute is used.
+  This optimization affects a minority of filesystems and operating systems.

doc/manual/src/release-notes/rl-next.md

@@ -1,2 +0,0 @@
-# Release X.Y (202?-??-??)
-

flake.nix

@@ -11,7 +11,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       version = lib.fileContents ./.version + versionSuffix;
       versionSuffix =
@@ -320,18 +320,12 @@
           };
           let
             canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform;
-
-            sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith {
-              filter = (path: type:
-                let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path);
-                in ! lib.any (re: builtins.match re relPath != null) rxs);
-              src = origSrc;
-            };
           in currentStdenv.mkDerivation (finalAttrs: {
             name = "nix-${version}";
             inherit version;
 
-            src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self;
+            src = self;
+
             VERSION_SUFFIX = versionSuffix;
 
             outputs = [ "out" "dev" "doc" ];

src/libcmd/common-eval-args.cc

@@ -165,7 +165,7 @@ Path lookupFileArg(EvalState & state, std::string_view s)
 {
     if (EvalSettings::isPseudoUrl(s)) {
         auto storePath = fetchers::downloadTarball(
-            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath;
+            state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath;
         return state.store->toRealPath(storePath);
     }
 

src/libcmd/installable-flake.cc

@@ -235,7 +235,7 @@ FlakeRef InstallableFlake::nixpkgsFlakeRef() const
         }
     }
 
-    return InstallableValue::nixpkgsFlakeRef();
+    return defaultNixpkgsFlakeRef();
 }
 
 }

src/libcmd/installable-flake.hh

@@ -67,9 +67,22 @@ struct InstallableFlake : InstallableValue
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
-    FlakeRef nixpkgsFlakeRef() const override;
+    FlakeRef nixpkgsFlakeRef() const;
 };
 
+/**
+ * Default flake ref for referring to Nixpkgs. For flakes that don't
+ * have their own Nixpkgs input, or other installables.
+ *
+ * It is a layer violation for Nix to know about Nixpkgs; currently just
+ * `nix develop` does. Be wary of using this /
+ * `InstallableFlake::nixpkgsFlakeRef` more places.
+ */
+static inline FlakeRef defaultNixpkgsFlakeRef()
+{
+    return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
+}
+
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
     std::shared_ptr<flake::LockedFlake> lockedFlake);

src/libcmd/installable-value.hh

@@ -96,11 +96,6 @@ struct InstallableValue : Installable
 
     UnresolvedApp toApp(EvalState & state);
 
-    virtual FlakeRef nixpkgsFlakeRef() const
-    {
-        return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
-    }
-
     static InstallableValue & require(Installable & installable);
     static ref<InstallableValue> require(ref<Installable> installable);
 };

src/libexpr/eval.cc

@@ -2525,7 +2525,7 @@ Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
     auto add = [&](const Path & p, const std::string & s = std::string()) {
-        if (pathExists(p)) {
+        if (pathAccessible(p)) {
             if (s.empty()) {
                 res.push_back(p);
             } else {

src/libexpr/parser.y

@@ -805,7 +805,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
     if (EvalSettings::isPseudoUrl(elem.second)) {
         try {
             auto storePath = fetchers::downloadTarball(
-                store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath;
+                store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).tree.storePath;
             res = { true, store->toRealPath(storePath) };
         } catch (FileTransferError & e) {
             logWarning({

src/libexpr/primops/fetchTree.cc

@@ -258,7 +258,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     //       https://github.com/NixOS/nix/issues/4313
     auto storePath =
         unpack
-        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
+        ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath
         : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
 
     if (expectedHash) {

src/libfetchers/attrs.hh

@@ -2,6 +2,7 @@
 ///@file
 
 #include "types.hh"
+#include "hash.hh"
 
 #include <variant>
 

src/libfetchers/fetchers.cc

@@ -159,6 +159,12 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
                 input.to_string(), *prevLastModified);
     }
 
+    if (auto prevRev = getRev()) {
+        if (input.getRev() != prevRev)
+            throw Error("'rev' attribute mismatch in input '%s', expected %s",
+                input.to_string(), prevRev->gitRev());
+    }
+
     if (auto prevRevCount = getRevCount()) {
         if (input.getRevCount() != prevRevCount)
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",

src/libfetchers/fetchers.hh

@@ -158,6 +158,7 @@ struct DownloadFileResult
     StorePath storePath;
     std::string etag;
     std::string effectiveUrl;
+    std::optional<std::string> immutableUrl;
 };
 
 DownloadFileResult downloadFile(
@@ -167,7 +168,14 @@ DownloadFileResult downloadFile(
     bool locked,
     const Headers & headers = {});
 
-std::pair<Tree, time_t> downloadTarball(
+struct DownloadTarballResult
+{
+    Tree tree;
+    time_t lastModified;
+    std::optional<std::string> immutableUrl;
+};
+
+DownloadTarballResult downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,

src/libfetchers/github.cc

@@ -207,21 +207,21 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
+        auto result = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
+        input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         getCache()->add(
             store,
             lockedAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", uint64_t(lastModified)}
+                {"lastModified", uint64_t(result.lastModified)}
             },
-            tree.storePath,
+            result.tree.storePath,
             true);
 
-        return {std::move(tree.storePath), input};
+        return {result.tree.storePath, input};
     }
 };
 

src/libfetchers/tarball.cc

@@ -32,7 +32,8 @@ DownloadFileResult downloadFile(
         return {
             .storePath = std::move(cached->storePath),
             .etag = getStrAttr(cached->infoAttrs, "etag"),
-            .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
+            .effectiveUrl = getStrAttr(cached->infoAttrs, "url"),
+            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
         };
     };
 
@@ -55,12 +56,14 @@ DownloadFileResult downloadFile(
     }
 
     // FIXME: write to temporary file.
-
     Attrs infoAttrs({
         {"etag", res.etag},
         {"url", res.effectiveUri},
     });
 
+    if (res.immutableUrl)
+        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+
     std::optional<StorePath> storePath;
 
     if (res.cached) {
@@ -107,10 +110,11 @@ DownloadFileResult downloadFile(
         .storePath = std::move(*storePath),
         .etag = res.etag,
         .effectiveUrl = res.effectiveUri,
+        .immutableUrl = res.immutableUrl,
     };
 }
 
-std::pair<Tree, time_t> downloadTarball(
+DownloadTarballResult downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -127,8 +131,9 @@ std::pair<Tree, time_t> downloadTarball(
 
     if (cached && !cached->expired)
         return {
-            Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
-            getIntAttr(cached->infoAttrs, "lastModified")
+            .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
+            .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
+            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
         };
 
     auto res = downloadFile(store, url, name, locked, headers);
@@ -156,6 +161,9 @@ std::pair<Tree, time_t> downloadTarball(
         {"etag", res.etag},
     });
 
+    if (res.immutableUrl)
+        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+
     getCache()->add(
         store,
         inAttrs,
@@ -164,8 +172,9 @@ std::pair<Tree, time_t> downloadTarball(
         locked);
 
     return {
-        Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
-        lastModified,
+        .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
+        .lastModified = lastModified,
+        .immutableUrl = res.immutableUrl,
     };
 }
 
@@ -185,21 +194,33 @@ struct CurlInputScheme : InputScheme
 
     virtual bool isValidURL(const ParsedURL & url) const = 0;
 
-    std::optional<Input> inputFromURL(const ParsedURL & url) const override
+    std::optional<Input> inputFromURL(const ParsedURL & _url) const override
     {
-        if (!isValidURL(url))
+        if (!isValidURL(_url))
             return std::nullopt;
 
         Input input;
 
-        auto urlWithoutApplicationScheme = url;
-        urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport;
+        auto url = _url;
+
+        url.scheme = parseUrlScheme(url.scheme).transport;
 
-        input.attrs.insert_or_assign("type", inputType());
-        input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string());
         auto narHash = url.query.find("narHash");
         if (narHash != url.query.end())
             input.attrs.insert_or_assign("narHash", narHash->second);
+
+        if (auto i = get(url.query, "rev"))
+            input.attrs.insert_or_assign("rev", *i);
+
+        if (auto i = get(url.query, "revCount"))
+            if (auto n = string2Int<uint64_t>(*i))
+                input.attrs.insert_or_assign("revCount", *n);
+
+        url.query.erase("rev");
+        url.query.erase("revCount");
+
+        input.attrs.insert_or_assign("type", inputType());
+        input.attrs.insert_or_assign("url", url.to_string());
         return input;
     }
 
@@ -208,7 +229,8 @@ struct CurlInputScheme : InputScheme
         auto type = maybeGetStrAttr(attrs, "type");
         if (type != inputType()) return {};
 
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"};
+        // FIXME: some of these only apply to TarballInputScheme.
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"};
         for (auto & [name, value] : attrs)
             if (!allowedNames.count(name))
                 throw Error("unsupported %s input attribute '%s'", *type, name);
@@ -271,10 +293,22 @@ struct TarballInputScheme : CurlInputScheme
                     : hasTarballExtension(url.path));
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
-        return {std::move(tree.storePath), input};
+        Input input(_input);
+        auto url = getStrAttr(input.attrs, "url");
+        auto result = downloadTarball(store, url, input.getName(), false);
+
+        if (result.immutableUrl) {
+            auto immutableInput = Input::fromURL(*result.immutableUrl);
+            // FIXME: would be nice to support arbitrary flakerefs
+            // here, e.g. git flakes.
+            if (immutableInput.getType() != "tarball")
+                throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported");
+            input = immutableInput;
+        }
+
+        return {result.tree.storePath, std::move(input)};
     }
 };
 

src/libstore/build/local-derivation-goal.cc

@@ -357,7 +357,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
         for (auto & [_, status] : initialOutputs) {
             if (!status.known) continue;
             if (buildMode != bmCheck && status.known->isValid()) continue;
-            auto p = worker.store.printStorePath(status.known->path);
+            auto p = worker.store.toRealPath(status.known->path);
             if (pathExists(chrootRootDir + p))
                 renameFile((chrootRootDir + p), p);
         }

src/libstore/derivations.cc

@@ -980,7 +980,8 @@ nlohmann::json DerivationOutput::toJSON(
 
 DerivationOutput DerivationOutput::fromJSON(
     const Store & store, std::string_view drvName, std::string_view outputName,
-    const nlohmann::json & _json)
+    const nlohmann::json & _json,
+    const ExperimentalFeatureSettings & xpSettings)
 {
     std::set<std::string_view> keys;
     auto json = (std::map<std::string, nlohmann::json>) _json;
@@ -1019,6 +1020,7 @@ DerivationOutput DerivationOutput::fromJSON(
     }
 
     else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
+        xpSettings.require(Xp::CaDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::CAFloating {
             .method = method,
@@ -1031,6 +1033,7 @@ DerivationOutput DerivationOutput::fromJSON(
     }
 
     else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
+        xpSettings.require(Xp::ImpureDerivations);
         auto [method, hashType] = methodAlgo();
         return DerivationOutput::Impure {
             .method = method,

src/libstore/derivations.hh

@@ -136,11 +136,15 @@ struct DerivationOutput : _DerivationOutputRaw
         const Store & store,
         std::string_view drvName,
         std::string_view outputName) const;
+    /**
+     * @param xpSettings Stop-gap to avoid globals during unit tests.
+     */
     static DerivationOutput fromJSON(
         const Store & store,
         std::string_view drvName,
         std::string_view outputName,
-        const nlohmann::json & json);
+        const nlohmann::json & json,
+        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
 };
 
 typedef std::map<std::string, DerivationOutput> DerivationOutputs;

src/libstore/derived-path.cc

@@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
 std::string DerivedPath::Built::to_string(const Store & store) const
 {
     return store.printStorePath(drvPath)
-        + "!"
+        + '^'
+        + outputs.to_string();
+}
+
+std::string DerivedPath::Built::to_string_legacy(const Store & store) const
+{
+    return store.printStorePath(drvPath)
+        + '!'
         + outputs.to_string();
 }
 
 std::string DerivedPath::to_string(const Store & store) const
 {
-    return std::visit(
-        [&](const auto & req) { return req.to_string(store); },
-        this->raw());
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & req) { return req.to_string(store); },
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
+}
+
+std::string DerivedPath::to_string_legacy(const Store & store) const
+{
+    return std::visit(overloaded {
+        [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); },
+        [&](const DerivedPath::Opaque & req) { return req.to_string(store); },
+    }, this->raw());
 }
 
 
@@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi
     };
 }
 
-DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
 {
-    size_t n = s.find("!");
+    size_t n = s.find(separator);
     return n == s.npos
         ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
         : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
 }
 
+DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "^");
+}
+
+DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
+{
+	return parseWith(store, s, "!");
+}
+
 RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
 {
     RealisedPath::Set res;

src/libstore/derived-path.hh

@@ -48,8 +48,18 @@ struct DerivedPathBuilt {
     StorePath drvPath;
     OutputsSpec outputs;
 
+    /**
+     * Uses `^` as the separator
+     */
     std::string to_string(const Store & store) const;
-    static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view);
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * The caller splits on the separator, so it works for both variants.
+     */
+    static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
     nlohmann::json toJSON(ref<Store> store) const;
 
     GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
@@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw {
         return static_cast<const Raw &>(*this);
     }
 
+    /**
+     * Uses `^` as the separator
+     */
     std::string to_string(const Store & store) const;
+    /**
+     * Uses `!` as the separator
+     */
+    std::string to_string_legacy(const Store & store) const;
+    /**
+     * Uses `^` as the separator
+     */
     static DerivedPath parse(const Store & store, std::string_view);
+    /**
+     * Uses `!` as the separator
+     */
+    static DerivedPath parseLegacy(const Store & store, std::string_view);
 };
 
 /**

src/libstore/filetransfer.cc

@@ -186,9 +186,9 @@ struct curlFileTransfer : public FileTransfer
             size_t realSize = size * nmemb;
             std::string line((char *) contents, realSize);
             printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line));
+
             static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
-            std::smatch match;
-            if (std::regex_match(line, match, statusLine)) {
+            if (std::smatch match; std::regex_match(line, match, statusLine)) {
                 result.etag = "";
                 result.data.clear();
                 result.bodySize = 0;
@@ -196,9 +196,11 @@ struct curlFileTransfer : public FileTransfer
                 acceptRanges = false;
                 encoding = "";
             } else {
+
                 auto i = line.find(':');
                 if (i != std::string::npos) {
                     std::string name = toLower(trim(line.substr(0, i)));
+
                     if (name == "etag") {
                         result.etag = trim(line.substr(i + 1));
                         /* Hack to work around a GitHub bug: it sends
@@ -212,10 +214,22 @@ struct curlFileTransfer : public FileTransfer
                             debug("shutting down on 200 HTTP response with expected ETag");
                             return 0;
                         }
-                    } else if (name == "content-encoding")
+                    }
+
+                    else if (name == "content-encoding")
                         encoding = trim(line.substr(i + 1));
+
                     else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes")
                         acceptRanges = true;
+
+                    else if (name == "link" || name == "x-amz-meta-link") {
+                        auto value = trim(line.substr(i + 1));
+                        static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase);
+                        if (std::smatch match; std::regex_match(value, match, linkRegex))
+                            result.immutableUrl = match.str(1);
+                        else
+                            debug("got invalid link header '%s'", value);
+                    }
                 }
             }
             return realSize;
@@ -345,7 +359,7 @@ struct curlFileTransfer : public FileTransfer
         {
             auto httpStatus = getHTTPStatus();
 
-            char * effectiveUriCStr;
+            char * effectiveUriCStr = nullptr;
             curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr);
             if (effectiveUriCStr)
                 result.effectiveUri = effectiveUriCStr;

src/libstore/filetransfer.hh

@@ -80,6 +80,10 @@ struct FileTransferResult
     std::string effectiveUri;
     std::string data;
     uint64_t bodySize = 0;
+    /* An "immutable" URL for this resource (i.e. one whose contents
+       will never change), as returned by the `Link: <url>;
+       rel="immutable"` header. */
+    std::optional<std::string> immutableUrl;
 };
 
 class Store;

src/libstore/gc.cc

@@ -563,7 +563,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                     /* On macOS, accepted sockets inherit the
                        non-blocking flag from the server socket, so
                        explicitly make it blocking. */
-                    if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1)
+                    if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1)
                         abort();
 
                     while (true) {

src/libstore/globals.cc

@@ -173,7 +173,7 @@ bool Settings::isWSL1()
 Path Settings::getDefaultSSLCertFile()
 {
     for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"})
-        if (pathExists(fn)) return fn;
+        if (pathAccessible(fn)) return fn;
     return "";
 }
 

src/libstore/remote-store.cc

@@ -90,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
 DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
 {
     auto s = readString(from);
-    return DerivedPath::parse(store, s);
+    return DerivedPath::parseLegacy(store, s);
 }
 
 void write(const Store & store, Sink & out, const DerivedPath & req)
 {
-    out << req.to_string(store);
+    out << req.to_string_legacy(store);
 }
 
 

src/libstore/tests/derivation.cc

@@ -1,6 +1,7 @@
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
+#include "experimental-features.hh"
 #include "derivations.hh"
 
 #include "tests/libstore.hh"
@@ -9,10 +10,32 @@ namespace nix {
 
 class DerivationTest : public LibStoreTest
 {
+public:
+    /**
+     * We set these in tests rather than the regular globals so we don't have
+     * to worry about race conditions if the tests run concurrently.
+     */
+    ExperimentalFeatureSettings mockXpSettings;
 };
 
-#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
-    TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) {    \
+class CaDerivationTest : public DerivationTest
+{
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "ca-derivations");
+    }
+};
+
+class ImpureDerivationTest : public DerivationTest
+{
+    void SetUp() override
+    {
+        mockXpSettings.set("experimental-features", "impure-derivations");
+    }
+};
+
+#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
+    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) {    \
         using nlohmann::literals::operator "" _json;           \
         ASSERT_EQ(                                             \
             STR ## _json,                                      \
@@ -22,7 +45,7 @@ class DerivationTest : public LibStoreTest
                 OUTPUT_NAME));                                 \
     }                                                          \
                                                                \
-    TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) {  \
+    TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) {  \
         using nlohmann::literals::operator "" _json;           \
         ASSERT_EQ(                                             \
             DerivationOutput { VAL },                          \
@@ -30,10 +53,11 @@ class DerivationTest : public LibStoreTest
                 *store,                                        \
                 DRV_NAME,                                      \
                 OUTPUT_NAME,                                   \
-                STR ## _json));                                \
+                STR ## _json,                                  \
+                mockXpSettings));                              \
     }
 
-TEST_JSON(inputAddressed,
+TEST_JSON(DerivationTest, inputAddressed,
     R"({
         "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
     })",
@@ -42,7 +66,7 @@ TEST_JSON(inputAddressed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(caFixed,
+TEST_JSON(DerivationTest, caFixed,
     R"({
         "hashAlgo": "r:sha256",
         "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
@@ -56,7 +80,7 @@ TEST_JSON(caFixed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(caFloating,
+TEST_JSON(CaDerivationTest, caFloating,
     R"({
         "hashAlgo": "r:sha256"
     })",
@@ -66,12 +90,12 @@ TEST_JSON(caFloating,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(deferred,
+TEST_JSON(DerivationTest, deferred,
     R"({ })",
     DerivationOutput::Deferred { },
     "drv-name", "output-name")
 
-TEST_JSON(impure,
+TEST_JSON(ImpureDerivationTest, impure,
     R"({
         "hashAlgo": "r:sha256",
         "impure": true

src/libstore/tests/derived-path.cc

@@ -51,6 +51,14 @@ TEST_F(DerivedPathTest, force_init)
 {
 }
 
+RC_GTEST_FIXTURE_PROP(
+    DerivedPathTest,
+    prop_legacy_round_rip,
+    (const DerivedPath & o))
+{
+    RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store)));
+}
+
 RC_GTEST_FIXTURE_PROP(
     DerivedPathTest,
     prop_round_rip,

src/libutil/config.cc

@@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings()
         set(s.first, s.second);
 }
 
-// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features.
-static bool applicable(const Config::SettingData & sd)
-{
-    return !sd.isAlias
-        && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature);
-}
-
 void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
 {
     for (auto & opt : _settings)
-        if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden))
+        if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
             res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
 }
 
@@ -154,7 +147,7 @@ nlohmann::json Config::toJSON()
 {
     auto res = nlohmann::json::object();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (!s.second.isAlias)
             res.emplace(s.first, s.second.setting->toJSON());
     return res;
 }
@@ -163,7 +156,7 @@ std::string Config::toKeyValue()
 {
     auto res = std::string();
     for (auto & s : _settings)
-        if (applicable(s.second))
+        if (s.second.isAlias)
             res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
     return res;
 }
@@ -171,9 +164,6 @@ std::string Config::toKeyValue()
 void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings) {
-        /* We do include args for settings gated on disabled
-           experimental-features. The args themselves however will also be
-           gated on any experimental feature the underlying setting is. */
         if (!s.second.isAlias)
             s.second.setting->convertToArg(args, category);
     }

src/libutil/tests/tests.cc

@@ -202,7 +202,7 @@ namespace nix {
     }
 
     TEST(pathExists, bogusPathDoesNotExist) {
-        ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes"));
+        ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes"));
     }
 
     /* ----------------------------------------------------------------------------

src/libutil/util.cc

@@ -263,6 +263,17 @@ bool pathExists(const Path & path)
     return false;
 }
 
+bool pathAccessible(const Path & path)
+{
+    try {
+        return pathExists(path);
+    } catch (SysError & e) {
+        // swallow EPERM
+        if (e.errNo == EPERM) return false;
+        throw;
+    }
+}
+
 
 Path readLink(const Path & path)
 {
@@ -1789,6 +1800,7 @@ void setStackSize(size_t stackSize)
 
 #if __linux__
 static AutoCloseFD fdSavedMountNamespace;
+static AutoCloseFD fdSavedRoot;
 #endif
 
 void saveMountNamespace()
@@ -1796,10 +1808,11 @@ void saveMountNamespace()
 #if __linux__
     static std::once_flag done;
     std::call_once(done, []() {
-        AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY);
-        if (!fd)
+        fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY);
+        if (!fdSavedMountNamespace)
             throw SysError("saving parent mount namespace");
-        fdSavedMountNamespace = std::move(fd);
+
+        fdSavedRoot = open("/proc/self/root", O_RDONLY);
     });
 #endif
 }
@@ -1812,9 +1825,16 @@ void restoreMountNamespace()
 
         if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1)
             throw SysError("restoring parent mount namespace");
-        if (chdir(savedCwd.c_str()) == -1) {
-            throw SysError("restoring cwd");
+
+        if (fdSavedRoot) {
+            if (fchdir(fdSavedRoot.get()))
+                throw SysError("chdir into saved root");
+            if (chroot("."))
+                throw SysError("chroot into saved root");
         }
+
+        if (chdir(savedCwd.c_str()) == -1)
+            throw SysError("restoring cwd");
     } catch (Error & e) {
         debug(e.msg());
     }

src/libutil/util.hh

@@ -119,6 +119,14 @@ struct stat lstat(const Path & path);
  */
 bool pathExists(const Path & path);
 
+/**
+ * A version of pathExists that returns false on a permission error.
+ * Useful for inferring default paths across directories that might not
+ * be readable.
+ * @return true iff the given path can be accessed and exists
+ */
+bool pathAccessible(const Path & path);
+
 /**
  * Read the contents (target) of a symbolic link.  The result is not
  * in any way canonicalised.

src/nix-collect-garbage/nix-collect-garbage.cc

@@ -77,7 +77,12 @@ static int main_nix_collect_garbage(int argc, char * * argv)
             return true;
         });
 
-        if (removeOld) removeOldGenerations(profilesDir());
+        if (removeOld) {
+            std::set<Path> dirsToClean = {
+                profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())};
+            for (auto & dir : dirsToClean)
+                removeOldGenerations(dir);
+        }
 
         // Run the actual garbage collector.
         if (!dryRun) {

src/nix/develop.cc

@@ -252,7 +252,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
     throw Error("get-env.sh failed to produce an environment");
 }
 
-struct Common : InstallableValueCommand, MixProfile
+struct Common : InstallableCommand, MixProfile
 {
     std::set<std::string> ignoreVars{
         "BASHOPTS",
@@ -374,7 +374,7 @@ struct Common : InstallableValueCommand, MixProfile
         return res;
     }
 
-    StorePath getShellOutPath(ref<Store> store, ref<InstallableValue> installable)
+    StorePath getShellOutPath(ref<Store> store, ref<Installable> installable)
     {
         auto path = installable->getStorePath();
         if (path && hasSuffix(path->to_string(), "-env"))
@@ -393,7 +393,7 @@ struct Common : InstallableValueCommand, MixProfile
     }
 
     std::pair<BuildEnvironment, std::string>
-    getBuildEnvironment(ref<Store> store, ref<InstallableValue> installable)
+    getBuildEnvironment(ref<Store> store, ref<Installable> installable)
     {
         auto shellOutPath = getShellOutPath(store, installable);
 
@@ -481,7 +481,7 @@ struct CmdDevelop : Common, MixEnvironment
           ;
     }
 
-    void run(ref<Store> store, ref<InstallableValue> installable) override
+    void run(ref<Store> store, ref<Installable> installable) override
     {
         auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable);
 
@@ -538,10 +538,14 @@ struct CmdDevelop : Common, MixEnvironment
             nixpkgsLockFlags.inputOverrides = {};
             nixpkgsLockFlags.inputUpdates = {};
 
+            auto nixpkgs = defaultNixpkgsFlakeRef();
+            if (auto * i = dynamic_cast<const InstallableFlake *>(&*installable))
+                nixpkgs = i->nixpkgsFlakeRef();
+
             auto bashInstallable = make_ref<InstallableFlake>(
                 this,
                 state,
-                installable->nixpkgsFlakeRef(),
+                std::move(nixpkgs),
                 "bashInteractive",
                 DefaultOutputs(),
                 Strings{},
@@ -605,7 +609,7 @@ struct CmdPrintDevEnv : Common, MixJSON
 
     Category category() override { return catUtility; }
 
-    void run(ref<Store> store, ref<InstallableValue> installable) override
+    void run(ref<Store> store, ref<Installable> installable) override
     {
         auto buildEnvironment = getBuildEnvironment(store, installable).first;
 

tests/ca/derivation-json.sh

@@ -16,6 +16,9 @@ drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json)
 # With --dry-run nothing is actually written
 [[ ! -e "$drvPath3" ]]
 
+# But the JSON is rejected without the experimental feature
+expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled"
+
 # Without --dry-run it is actually written
 drvPath4=$(nix derivation add < $TEST_HOME/foo.json)
 [[ "$drvPath4" = "$drvPath3" ]]

tests/experimental-features.sh

@@ -1,25 +1,27 @@
 source common.sh
 
-# Without flakes, flake options should not show up
-# With flakes, flake options should show up
-
-function both_ways {
-    nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake
-    nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake
-
-    # Also, the order should not matter
-    nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake
-    nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake
-}
-
-# Simple case, the configuration effects the running command
-both_ways show-config
-
-# Skipping for now, because we actually *do* want these to show up in
-# the manual, just be marked experimental. Will reenable once the manual
-# generation takes advantage of the JSON metadata on this.
-
-# both_ways store gc --help
+# Skipping these two for now, because we actually *do* want flags and
+# config settings to always show up in the manual, just be marked
+# experimental. Will reenable once the manual generation takes advantage
+# of the JSON metadata on this.
+#
+# # Without flakes, flake options should not show up
+# # With flakes, flake options should show up
+#
+# function grep_both_ways {
+#     nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake
+#     nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake
+#
+#     # Also, the order should not matter
+#     nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake
+#     nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake
+# }
+#
+# # Simple case, the configuration effects the running command
+# grep_both_ways show-config
+#
+# # Medium case, the configuration effects --help
+# grep_both_ways store gc --help
 
 expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no'
 nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no'

tests/gc.sh

@@ -52,9 +52,7 @@ rmdir $NIX_STORE_DIR/.links
 rmdir $NIX_STORE_DIR
 
 ## Test `nix-collect-garbage -d`
-# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
-# using them
-if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+testCollectGarbageD () {
     clearProfiles
     # Run two `nix-env` commands, should create two generations of
     # the profile
@@ -66,4 +64,17 @@ if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
     # left
     nix-collect-garbage -d
     [[ $(nix-env --list-generations | wc -l) -eq 1 ]]
+}
+# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're
+# using them
+if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then
+    testCollectGarbageD
+
+    # Run the same test, but forcing the profiles at their legacy location under
+    # /nix/var/nix.
+    #
+    # Regression test for #8294
+    rm ~/.nix-profile
+    ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile
+    testCollectGarbageD
 fi

tests/impure-derivations.sh

@@ -10,6 +10,15 @@ clearStore
 # Basic test of impure derivations: building one a second time should not use the previous result.
 printf 0 > $TEST_ROOT/counter
 
+# `nix derivation add` with impure derivations work
+drvPath=$(nix-instantiate ./impure-derivations.nix -A impure)
+nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json
+drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json)
+[[ "$drvPath" = "$drvPath2" ]]
+
+# But only with the experimental feature!
+expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled"
+
 nix build --dry-run --json --file ./impure-derivations.nix impure.all
 json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all)
 path1=$(echo $json | jq -r .[].outputs.out)

tests/nix-shell.sh

@@ -98,6 +98,18 @@ nix develop -f "$shellDotNix" shellDrv -c echo foo |& grepQuiet foo
 nix print-dev-env -f "$shellDotNix" shellDrv > $TEST_ROOT/dev-env.sh
 nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json
 
+# Test with raw drv
+
+shellDrv=$(nix-instantiate "$shellDotNix" -A shellDrv.out)
+
+nix develop $shellDrv -c bash -c '[[ -n $stdenv ]]'
+
+nix print-dev-env $shellDrv > $TEST_ROOT/dev-env2.sh
+nix print-dev-env $shellDrv --json > $TEST_ROOT/dev-env2.json
+
+diff $TEST_ROOT/dev-env{,2}.sh
+diff $TEST_ROOT/dev-env{,2}.json
+
 # Ensure `nix print-dev-env --json` contains variable assignments.
 [[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]]