Merge pull request #14238 from NixOS/backport-14237-to-2.32-maintenance

[Backport 2.32-maintenance] Remove validation of URLs passed to FileTransferRequest verbatim

.version

@@ -1 +1 @@
-2.32.0
+2.32.1

flake.nix

@@ -32,7 +32,7 @@
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux32BitSystems = [ "i686-linux" ];
       linux64BitSystems = [

nix-meson-build-support/common/assert-fail/meson.build

@@ -0,0 +1,32 @@
+can_wrap_assert_fail_test_code = '''
+#include <cstdlib>
+#include <cassert>
+
+int main()
+{
+    assert(0);
+}
+
+extern "C" void * __real___assert_fail(const char *, const char *, unsigned int, const char *);
+
+extern "C" void *
+__wrap___assert_fail(const char *, const char *, unsigned int, const char *)
+{
+    return __real___assert_fail(nullptr, nullptr, 0, nullptr);
+}
+'''
+
+wrap_assert_fail_args = [ '-Wl,--wrap=__assert_fail' ]
+
+can_wrap_assert_fail = cxx.links(
+  can_wrap_assert_fail_test_code,
+  args : wrap_assert_fail_args,
+  name : 'linker can wrap __assert_fail',
+)
+
+if can_wrap_assert_fail
+  deps_other += declare_dependency(
+    sources : 'wrap-assert-fail.cc',
+    link_args : wrap_assert_fail_args,
+  )
+endif

nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc

@@ -0,0 +1,17 @@
+#include "nix/util/error.hh"
+
+#include <cstdio>
+#include <cstdlib>
+#include <cinttypes>
+#include <string_view>
+
+extern "C" [[noreturn]] void __attribute__((weak))
+__wrap___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function)
+{
+    char buf[512];
+    int n =
+        snprintf(buf, sizeof(buf), "Assertion '%s' failed in %s at %s:%" PRIuLEAST32, assertion, function, file, line);
+    if (n < 0)
+        nix::panic("Assertion failed and could not format error message");
+    nix::panic(std::string_view(buf, std::min(static_cast<int>(sizeof(buf)), n)));
+}

nix-meson-build-support/common/meson.build

@@ -44,3 +44,5 @@ endif
 
 # Darwin ld doesn't like "X.Y.Zpre"
 nix_soversion = meson.project_version().split('pre')[0]
+
+subdir('assert-fail')

packaging/dependencies.nix

@@ -57,15 +57,20 @@ scope: {
         prevAttrs.postInstall;
   });
 
-  toml11 = pkgs.toml11.overrideAttrs rec {
-    version = "4.4.0";
-    src = pkgs.fetchFromGitHub {
-      owner = "ToruNiina";
-      repo = "toml11";
-      tag = "v${version}";
-      hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
-    };
-  };
+  # TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release
+  toml11 =
+    if lib.versionAtLeast pkgs.toml11.version "4.4.0" then
+      pkgs.toml11
+    else
+      pkgs.toml11.overrideAttrs rec {
+        version = "4.4.0";
+        src = pkgs.fetchFromGitHub {
+          owner = "ToruNiina";
+          repo = "toml11";
+          tag = "v${version}";
+          hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ=";
+        };
+      };
 
   # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
   boost =

src/libfetchers/tarball.cc

@@ -42,7 +42,7 @@ DownloadFileResult downloadFile(
     if (cached && !cached->expired)
         return useCached();
 
-    FileTransferRequest request(ValidURL{url});
+    FileTransferRequest request(VerbatimURL{url});
     request.headers = headers;
     if (cached)
         request.expectedETag = getStrAttr(cached->value, "etag");
@@ -107,13 +107,13 @@ DownloadFileResult downloadFile(
 static DownloadTarballResult downloadTarball_(
     const Settings & settings, const std::string & urlS, const Headers & headers, const std::string & displayPrefix)
 {
-    ValidURL url = urlS;
+    ParsedURL url = parseURL(urlS);
 
     // Some friendly error messages for common mistakes.
     // Namely lets catch when the url is a local file path, but
     // it is not in fact a tarball.
-    if (url.scheme() == "file") {
-        std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path());
+    if (url.scheme == "file") {
+        std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path);
         if (!exists(localPath)) {
             throw Error("tarball '%s' does not exist.", localPath);
         }
@@ -164,7 +164,7 @@ static DownloadTarballResult downloadTarball_(
 
     /* Note: if the download is cached, `importTarball()` will receive
        no data, which causes it to import an empty tarball. */
-    auto archive = !url.path().empty() && hasSuffix(toLower(url.path().back()), ".zip") ? ({
+    auto archive = !url.path.empty() && hasSuffix(toLower(url.path.back()), ".zip") ? ({
         /* In streaming mode, libarchive doesn't handle
            symlinks in zip files correctly (#10649). So write
            the entire file to disk so libarchive can access it
@@ -178,7 +178,7 @@ static DownloadTarballResult downloadTarball_(
         }
         TarArchive{path};
     })
-                                                                                        : TarArchive{*source};
+                                                                                    : TarArchive{*source};
     auto tarballCache = getTarballCache();
     auto parseSink = tarballCache->getFileSystemObjectSink();
     auto lastModified = unpackTarfileToSink(archive, *parseSink);

src/libstore/builtins/fetchurl.cc

@@ -37,7 +37,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx)
 
     auto fetch = [&](const std::string & url) {
         auto source = sinkToSource([&](Sink & sink) {
-            FileTransferRequest request(ValidURL{url});
+            FileTransferRequest request(VerbatimURL{url});
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);

src/libstore/derivation-options.cc

@@ -99,6 +99,17 @@ DerivationOptions DerivationOptions::fromStructuredAttrs(
     return fromStructuredAttrs(env, parsed ? &*parsed : nullptr);
 }
 
+static void flatten(const nlohmann::json & value, StringSet & res)
+{
+    if (value.is_array())
+        for (auto & v : value)
+            flatten(v, res);
+    else if (value.is_string())
+        res.insert(value);
+    else
+        throw Error("'exportReferencesGraph' value is not an array or a string");
+}
+
 DerivationOptions
 DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn)
 {
@@ -219,12 +230,9 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt
                     if (!e || !e->is_object())
                         return ret;
                     for (auto & [key, value] : getObject(*e)) {
-                        if (value.is_array())
-                            ret.insert_or_assign(key, value);
-                        else if (value.is_string())
-                            ret.insert_or_assign(key, StringSet{value});
-                        else
-                            throw Error("'exportReferencesGraph' value is not an array or a string");
+                        StringSet ss;
+                        flatten(value, ss);
+                        ret.insert_or_assign(key, std::move(ss));
                     }
                 } else {
                     auto s = getOr(env, "exportReferencesGraph", "");

src/libstore/include/nix/store/filetransfer.hh

@@ -79,7 +79,7 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT;
 
 struct FileTransferRequest
 {
-    ValidURL uri;
+    VerbatimURL uri;
     Headers headers;
     std::string expectedETag;
     bool verifyTLS = true;
@@ -93,7 +93,7 @@ struct FileTransferRequest
     std::string mimeType;
     std::function<void(std::string_view data)> dataCallback;
 
-    FileTransferRequest(ValidURL uri)
+    FileTransferRequest(VerbatimURL uri)
         : uri(std::move(uri))
         , parentAct(getCurActivity())
     {

src/libstore/local-store.cc

@@ -1383,7 +1383,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
         for (auto & link : DirectoryIterator{linksDir}) {
             checkInterrupt();
             auto name = link.path().filename();
-            printMsg(lvlTalkative, "checking contents of '%s'", name);
+            printMsg(lvlTalkative, "checking contents of %s", name);
             PosixSourceAccessor accessor;
             std::string hash = hashPath(
                                    PosixSourceAccessor::createAtRoot(link.path()),
@@ -1391,10 +1391,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
                                    HashAlgorithm::SHA256)
                                    .first.to_string(HashFormat::Nix32, false);
             if (hash != name.string()) {
-                printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash);
+                printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash);
                 if (repair) {
                     std::filesystem::remove(link.path());
-                    printInfo("removed link '%s'", link.path());
+                    printInfo("removed link %s", link.path());
                 } else {
                     errors = true;
                 }

src/libstore/optimise-store.cc

@@ -202,7 +202,7 @@ void LocalStore::optimisePath_(
                    full.  When that happens, it's fine to ignore it: we
                    just effectively disable deduplication of this
                    file.  */
-                printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno));
+                printInfo("cannot link %s to '%s': %s", linkPath, path, strerror(errno));
                 return;
             }
 
@@ -216,11 +216,11 @@ void LocalStore::optimisePath_(
     auto stLink = lstat(linkPath.string());
 
     if (st.st_ino == stLink.st_ino) {
-        debug("'%1%' is already linked to '%2%'", path, linkPath);
+        debug("'%1%' is already linked to %2%", path, linkPath);
         return;
     }
 
-    printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath);
+    printMsg(lvlTalkative, "linking '%1%' to %2%", path, linkPath);
 
     /* Make the containing directory writable, but only if it's not
        the store itself (we don't want or need to mess with its
@@ -245,7 +245,7 @@ void LocalStore::optimisePath_(
                systems).  This is likely to happen with empty files.
                Just shrug and ignore. */
             if (st.st_size)
-                printInfo("'%1%' has maximum number of links", linkPath);
+                printInfo("%1% has maximum number of links", linkPath);
             return;
         }
         throw;
@@ -256,13 +256,13 @@ void LocalStore::optimisePath_(
         std::filesystem::rename(tempLink, path);
     } catch (std::filesystem::filesystem_error & e) {
         std::filesystem::remove(tempLink);
-        printError("unable to unlink '%1%'", tempLink);
+        printError("unable to unlink %1%", tempLink);
         if (e.code() == std::errc::too_many_links) {
             /* Some filesystems generate too many links on the rename,
                rather than on the original link.  (Probably it
                temporarily increases the st_nlink field before
                decreasing it again.) */
-            debug("'%s' has reached maximum number of links", linkPath);
+            debug("%s has reached maximum number of links", linkPath);
             return;
         }
         throw;

src/libutil-tests/url.cc

@@ -868,6 +868,12 @@ TEST_P(ParsedURLPathSegmentsTest, segmentsAreCorrect)
     EXPECT_EQ(encodeUrlPath(segments), testCase.path);
 }
 
+TEST_P(ParsedURLPathSegmentsTest, to_string)
+{
+    const auto & testCase = GetParam();
+    EXPECT_EQ(testCase.url, parseURL(testCase.url).to_string());
+}
+
 INSTANTIATE_TEST_SUITE_P(
     ParsedURL,
     ParsedURLPathSegmentsTest,
@@ -886,6 +892,13 @@ INSTANTIATE_TEST_SUITE_P(
             .skipEmpty = false,
             .description = "empty_authority_empty_path",
         },
+        ParsedURLPathSegmentsTestCase{
+            .url = "path:/",
+            .segments = {"", ""},
+            .path = "/",
+            .skipEmpty = false,
+            .description = "empty_authority_root_path",
+        },
         ParsedURLPathSegmentsTestCase{
             .url = "scheme:///",
             .segments = {"", ""},

src/libutil/include/nix/util/url.hh

@@ -6,6 +6,9 @@
 
 #include "nix/util/error.hh"
 #include "nix/util/canon-path.hh"
+#include "nix/util/split.hh"
+#include "nix/util/util.hh"
+#include "nix/util/variant-wrapper.hh"
 
 namespace nix {
 
@@ -342,8 +345,7 @@ ParsedURL fixGitURL(const std::string & url);
 bool isValidSchemeName(std::string_view scheme);
 
 /**
- * Either a ParsedURL or a verbatim string, but the string must be a valid
- * ParsedURL. This is necessary because in certain cases URI must be passed
+ * Either a ParsedURL or a verbatim string. This is necessary because in certain cases URI must be passed
  * verbatim (e.g. in builtin fetchers), since those are specified by the user.
  * In those cases normalizations performed by the ParsedURL might be surprising
  * and undesirable, since Nix must be a universal client that has to work with
@@ -354,23 +356,23 @@ bool isValidSchemeName(std::string_view scheme);
  *
  * Though we perform parsing and validation for internal needs.
  */
-struct ValidURL : private ParsedURL
+struct VerbatimURL
 {
-    std::optional<std::string> encoded;
+    using Raw = std::variant<std::string, ParsedURL>;
+    Raw raw;
 
-    ValidURL(std::string str)
-        : ParsedURL(parseURL(str, /*lenient=*/false))
-        , encoded(std::move(str))
+    VerbatimURL(std::string_view s)
+        : raw(std::string{s})
     {
     }
 
-    ValidURL(std::string_view str)
-        : ValidURL(std::string{str})
+    VerbatimURL(std::string s)
+        : raw(std::move(s))
     {
     }
 
-    ValidURL(ParsedURL parsed)
-        : ParsedURL{std::move(parsed)}
+    VerbatimURL(ParsedURL url)
+        : raw(std::move(url))
     {
     }
 
@@ -379,25 +381,35 @@ struct ValidURL : private ParsedURL
      */
     std::string to_string() const
     {
-        return encoded.or_else([&]() -> std::optional<std::string> { return ParsedURL::to_string(); }).value();
+        return std::visit(
+            overloaded{
+                [](const std::string & str) { return str; }, [](const ParsedURL & url) { return url.to_string(); }},
+            raw);
     }
 
-    const ParsedURL & parsed() const &
+    const ParsedURL parsed() const
     {
-        return *this;
+        return std::visit(
+            overloaded{
+                [](const std::string & str) { return parseURL(str); }, [](const ParsedURL & url) { return url; }},
+            raw);
     }
 
     std::string_view scheme() const &
     {
-        return ParsedURL::scheme;
-    }
-
-    const auto & path() const &
-    {
-        return ParsedURL::path;
+        return std::visit(
+            overloaded{
+                [](std::string_view str) {
+                    auto scheme = splitPrefixTo(str, ':');
+                    if (!scheme)
+                        throw BadURL("URL '%s' doesn't have a scheme", str);
+                    return *scheme;
+                },
+                [](const ParsedURL & url) -> std::string_view { return url.scheme; }},
+            raw);
     }
 };
 
-std::ostream & operator<<(std::ostream & os, const ValidURL & url);
+std::ostream & operator<<(std::ostream & os, const VerbatimURL & url);
 
 } // namespace nix

src/libutil/url.cc

@@ -350,7 +350,7 @@ std::string ParsedURL::renderAuthorityAndPath() const
            must either be empty or begin with a slash ("/") character. */
         assert(path.empty() || path.front().empty());
         res += authority->to_string();
-    } else if (std::ranges::equal(std::views::take(path, 2), std::views::repeat("", 2))) {
+    } else if (std::ranges::equal(std::views::take(path, 3), std::views::repeat("", 3))) {
         /* If a URI does not contain an authority component, then the path cannot begin
            with two slash characters ("//") */
         unreachable();
@@ -434,7 +434,7 @@ bool isValidSchemeName(std::string_view s)
     return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default);
 }
 
-std::ostream & operator<<(std::ostream & os, const ValidURL & url)
+std::ostream & operator<<(std::ostream & os, const VerbatimURL & url)
 {
     os << url.to_string();
     return os;

src/nix/prefetch.cc

@@ -105,7 +105,7 @@ std::tuple<StorePath, Hash> prefetchFile(
 
             FdSink sink(fd.get());
 
-            FileTransferRequest req(ValidURL{url});
+            FileTransferRequest req(VerbatimURL{url});
             req.decompress = false;
             getFileTransfer()->download(std::move(req), sink);
         }

tests/functional/fetchurl.sh

@@ -88,8 +88,3 @@ requireDaemonNewerThan "2.20"
 expected=100
 if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly
 expectStderr $expected nix-build --expr '{ url }: builtins.derivation { name = "nix-cache-info"; system = "x86_64-linux"; builder = "builtin:fetchurl"; inherit url; outputHashMode = "flat"; }' --argstr url "file://$narxz" 2>&1 | grep 'must be a fixed-output or impure derivation'
-
-requireDaemonNewerThan "2.32.0pre20250831"
-
-expect 1 nix-build --expr 'import <nix/fetchurl.nix>' --argstr name 'name' --argstr url "file://authority.not.allowed/fetchurl.sh?a=1&a=2" --no-out-link |&
-        grepQuiet "error: file:// URL 'file://authority.not.allowed/fetchurl.sh?a=1&a=2' has unexpected authority 'authority.not.allowed'"

tests/functional/structured-attrs.nix

@@ -82,4 +82,8 @@ mkDerivation {
   "foo$" = "BAD";
 
   exportReferencesGraph.refs = [ dep ];
+  exportReferencesGraph.refs2 = [
+    dep
+    [ dep ]
+  ]; # regression test for heterogeneous arrays
 }

tests/functional/structured-attrs.sh

@@ -2,9 +2,8 @@
 
 source common.sh
 
-# 27ce722638 required some incompatible changes to the nix file, so skip this
-# tests for the older versions
-requireDaemonNewerThan "2.4pre20210712"
+# https://github.com/NixOS/nix/pull/14189
+requireDaemonNewerThan "2.33"
 
 clearStoreIfPossible