Remove meta.maintainers

Some of the maintainer attribute names got changed in nixos-unstable
(e.g. "edolstra" is now "eelco") but we want this flake to work on
nixos-24.11. So just get rid of them.

(cherry picked from commit 93d8f62057)
(cherry picked from commit b3b4fc21da)

.version

@@ -1 +1 @@
-2.26.0
+2.26.4

default.nix

@@ -1,10 +1,9 @@
-(import
-  (
-    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
-    fetchTarball {
-      url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
-      sha256 = lock.nodes.flake-compat.locked.narHash;
-    }
-  )
-  { src = ./.; }
-).defaultNix
+(import (
+  let
+    lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+  in
+  fetchTarball {
+    url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+    sha256 = lock.nodes.flake-compat.locked.narHash;
+  }
+) { src = ./.; }).defaultNix

doc/manual/generate-builtins.nix

@@ -5,7 +5,15 @@ in
 
 builtinsInfo:
 let
-  showBuiltin = name: { doc, type ? null, args ? [ ], experimental-feature ? null, impure-only ? false }:
+  showBuiltin =
+    name:
+    {
+      doc,
+      type ? null,
+      args ? [ ],
+      experimental-feature ? null,
+      impure-only ? false,
+    }:
     let
       type' = optionalString (type != null) " (${type})";
 

doc/manual/generate-manpage.nix

@@ -32,7 +32,13 @@ let
 
   commandInfo = fromJSON commandDump;
 
-  showCommand = { command, details, filename, toplevel }:
+  showCommand =
+    {
+      command,
+      details,
+      filename,
+      toplevel,
+    }:
     let
 
       result = ''
@@ -56,26 +62,27 @@ let
         ${maybeOptions}
       '';
 
-      showSynopsis = command: args:
+      showSynopsis =
+        command: args:
         let
-          showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "...";
+          showArgument = arg: "*${arg.label}*" + optionalString (!arg ? arity) "...";
           arguments = concatStringsSep " " (map showArgument args);
-        in ''
+        in
+        ''
           `${command}` [*option*...] ${arguments}
         '';
 
-      maybeSubcommands = optionalString (details ? commands && details.commands != {})
-        ''
-          where *subcommand* is one of the following:
+      maybeSubcommands = optionalString (details ? commands && details.commands != { }) ''
+        where *subcommand* is one of the following:
 
-          ${subcommands}
-        '';
+        ${subcommands}
+      '';
 
-      subcommands = if length categories > 1
-        then listCategories
-        else listSubcommands details.commands;
+      subcommands = if length categories > 1 then listCategories else listSubcommands details.commands;
 
-      categories = sort (x: y: x.id < y.id) (unique (map (cmd: cmd.category) (attrValues details.commands)));
+      categories = sort (x: y: x.id < y.id) (
+        unique (map (cmd: cmd.category) (attrValues details.commands))
+      );
 
       listCategories = concatStrings (map showCategory categories);
 
@@ -99,38 +106,39 @@ let
 
             ${allStores}
           '';
-          index = replaceStrings
-            [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ]
-            [ storesOverview "#local-store" "#local-daemon-store" ]
-            details.doc;
+          index =
+            replaceStrings
+              [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ]
+              [ storesOverview "#local-store" "#local-daemon-store" ]
+              details.doc;
           storesOverview =
             let
-              showEntry = store:
-                "- [${store.name}](#${store.slug})";
+              showEntry = store: "- [${store.name}](#${store.slug})";
             in
             concatStringsSep "\n" (map showEntry storesList) + "\n";
           allStores = concatStringsSep "\n" (attrValues storePages);
-          storePages = listToAttrs
-            (map (s: { name = s.filename; value = s.page; }) storesList);
+          storePages = listToAttrs (
+            map (s: {
+              name = s.filename;
+              value = s.page;
+            }) storesList
+          );
           storesList = showStoreDocs {
             storeInfo = commandInfo.stores;
             inherit inlineHTML;
           };
-          hasInfix = infix: content:
+          hasInfix =
+            infix: content:
             builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content);
         in
         optionalString (details ? doc) (
           # An alternate implementation with builtins.match stack overflowed on some systems.
-          if hasInfix "@store-types@" details.doc
-          then help-stores
-          else details.doc
+          if hasInfix "@store-types@" details.doc then help-stores else details.doc
         );
 
       maybeOptions =
         let
-          allVisibleOptions = filterAttrs
-            (_: o: ! o.hiddenCategory)
-            (details.flags // toplevel.flags);
+          allVisibleOptions = filterAttrs (_: o: !o.hiddenCategory) (details.flags // toplevel.flags);
         in
         optionalString (allVisibleOptions != { }) ''
           # Options
@@ -142,55 +150,73 @@ let
           > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags.
         '';
 
-      showOptions = inlineHTML: allOptions:
+      showOptions =
+        inlineHTML: allOptions:
         let
           showCategory = cat: opts: ''
             ${optionalString (cat != "") "## ${cat}"}
 
             ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))}
           '';
-          showOption = name: option:
+          showOption =
+            name: option:
             let
               result = trim ''
                 - ${item}
 
                   ${option.description}
               '';
-              item = if inlineHTML
-                then ''<span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}''
-                else "`--${name}` ${shortName} ${labels}";
-              shortName = optionalString
-                (option ? shortName)
-                ("/ `-${option.shortName}`");
-              labels = optionalString
-                (option ? labels)
-                (concatStringsSep " " (map (s: "*${s}*") option.labels));
-            in result;
-          categories = mapAttrs
-            # Convert each group from a list of key-value pairs back to an attrset
-            (_: listToAttrs)
-            (groupBy
-              (cmd: cmd.value.category)
-              (attrsToList allOptions));
-        in concatStrings (attrValues (mapAttrs showCategory categories));
-    in squash result;
+              item =
+                if inlineHTML then
+                  ''<span id="opt-${name}">[`--${name}`](#opt-${name})</span> ${shortName} ${labels}''
+                else
+                  "`--${name}` ${shortName} ${labels}";
+              shortName = optionalString (option ? shortName) ("/ `-${option.shortName}`");
+              labels = optionalString (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels));
+            in
+            result;
+          categories =
+            mapAttrs
+              # Convert each group from a list of key-value pairs back to an attrset
+              (_: listToAttrs)
+              (groupBy (cmd: cmd.value.category) (attrsToList allOptions));
+        in
+        concatStrings (attrValues (mapAttrs showCategory categories));
+    in
+    squash result;
 
   appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name;
 
-  processCommand = { command, details, filename, toplevel }:
+  processCommand =
+    {
+      command,
+      details,
+      filename,
+      toplevel,
+    }:
     let
       cmd = {
         inherit command;
         name = filename + ".md";
-        value = showCommand { inherit command details filename toplevel; };
+        value = showCommand {
+          inherit
+            command
+            details
+            filename
+            toplevel
+            ;
+        };
       };
-      subcommand = subCmd: processCommand {
-        command = command + " " + subCmd;
-        details = details.commands.${subCmd};
-        filename = appendName filename subCmd;
-        inherit toplevel;
-      };
-    in [ cmd ] ++ concatMap subcommand (attrNames details.commands or {});
+      subcommand =
+        subCmd:
+        processCommand {
+          command = command + " " + subCmd;
+          details = details.commands.${subCmd};
+          filename = appendName filename subCmd;
+          inherit toplevel;
+        };
+    in
+    [ cmd ] ++ concatMap subcommand (attrNames details.commands or { });
 
   manpages = processCommand {
     command = "nix";
@@ -199,9 +225,11 @@ let
     toplevel = commandInfo.args;
   };
 
-  tableOfContents = let
-    showEntry = page:
-      "    - [${page.command}](command-ref/new-cli/${page.name})";
-    in concatStringsSep "\n" (map showEntry manpages) + "\n";
+  tableOfContents =
+    let
+      showEntry = page: "    - [${page.command}](command-ref/new-cli/${page.name})";
+    in
+    concatStringsSep "\n" (map showEntry manpages) + "\n";
 
-in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }
+in
+(listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; }

doc/manual/generate-settings.nix

@@ -1,67 +1,99 @@
 let
-  inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs;
-  inherit (import <nix/utils.nix>) concatStrings indent optionalString squash;
+  inherit (builtins)
+    attrValues
+    concatStringsSep
+    isAttrs
+    isBool
+    mapAttrs
+    ;
+  inherit (import <nix/utils.nix>)
+    concatStrings
+    indent
+    optionalString
+    squash
+    ;
 in
 
 # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown`
-{ prefix, inlineHTML ? true }: settingsInfo:
+{
+  prefix,
+  inlineHTML ? true,
+}:
+settingsInfo:
 
 let
 
-  showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }:
+  showSetting =
+    prefix: setting:
+    {
+      description,
+      documentDefault,
+      defaultValue,
+      aliases,
+      value,
+      experimentalFeature,
+    }:
     let
       result = squash ''
-          - ${item}
+        - ${item}
 
-          ${indent "  " body}
-        '';
-      item = if inlineHTML
-        then ''<span id="${prefix}-${setting}">[`${setting}`](#${prefix}-${setting})</span>''
-        else "`${setting}`";
+        ${indent "  " body}
+      '';
+      item =
+        if inlineHTML then
+          ''<span id="${prefix}-${setting}">[`${setting}`](#${prefix}-${setting})</span>''
+        else
+          "`${setting}`";
       # separate body to cleanly handle indentation
       body = ''
-          ${experimentalFeatureNote}
+        ${experimentalFeatureNote}
 
-          ${description}
+        ${description}
 
-          **Default:** ${showDefault documentDefault defaultValue}
+        **Default:** ${showDefault documentDefault defaultValue}
 
-          ${showAliases aliases}
-        '';
+        ${showAliases aliases}
+      '';
 
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
-          > **Warning**
-          >
-          > This setting is part of an
-          > [experimental feature](@docroot@/development/experimental-features.md).
-          >
-          > To change this setting, make sure the
-          > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature})
-          > is enabled.
-          > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
-          >
-          > ```
-          > extra-experimental-features = ${experimentalFeature}
-          > ${setting} = ...
-          > ```
-        '';
+        > **Warning**
+        >
+        > This setting is part of an
+        > [experimental feature](@docroot@/development/experimental-features.md).
+        >
+        > To change this setting, make sure the
+        > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature})
+        > is enabled.
+        > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md):
+        >
+        > ```
+        > extra-experimental-features = ${experimentalFeature}
+        > ${setting} = ...
+        > ```
+      '';
 
-      showDefault = documentDefault: defaultValue:
+      showDefault =
+        documentDefault: defaultValue:
         if documentDefault then
           # a StringMap value type is specified as a string, but
           # this shows the value type. The empty stringmap is `null` in
           # JSON, but that converts to `{ }` here.
-          if defaultValue == "" || defaultValue == [] || isAttrs defaultValue
-            then "*empty*"
-            else if isBool defaultValue then
-              if defaultValue then "`true`" else "`false`"
-            else "`${toString defaultValue}`"
-        else "*machine-specific*";
+          if defaultValue == "" || defaultValue == [ ] || isAttrs defaultValue then
+            "*empty*"
+          else if isBool defaultValue then
+            if defaultValue then "`true`" else "`false`"
+          else
+            "`${toString defaultValue}`"
+        else
+          "*machine-specific*";
 
-      showAliases = aliases:
-          optionalString (aliases != [])
-            "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
+      showAliases =
+        aliases:
+        optionalString (aliases != [ ])
+          "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}";
 
-    in result;
+    in
+    result;
 
-in concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo))
+in
+concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo))

doc/manual/generate-store-info.nix

@@ -1,6 +1,20 @@
 let
-  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
-  inherit (import <nix/utils.nix>) optionalString filterAttrs trim squash toLower unique indent;
+  inherit (builtins)
+    attrNames
+    listToAttrs
+    concatStringsSep
+    readFile
+    replaceStrings
+    ;
+  inherit (import <nix/utils.nix>)
+    optionalString
+    filterAttrs
+    trim
+    squash
+    toLower
+    unique
+    indent
+    ;
   showSettings = import <nix/generate-settings.nix>;
 in
 
@@ -14,7 +28,13 @@ in
 
 let
 
-  showStore = { name, slug }: { settings, doc, experimentalFeature }:
+  showStore =
+    { name, slug }:
+    {
+      settings,
+      doc,
+      experimentalFeature,
+    }:
     let
       result = squash ''
         # ${name}
@@ -25,7 +45,10 @@ let
 
         ## Settings
 
-        ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings}
+        ${showSettings {
+          prefix = "store-${slug}";
+          inherit inlineHTML;
+        } settings}
       '';
 
       experimentalFeatureNote = optionalString (experimentalFeature != null) ''
@@ -43,15 +66,15 @@ let
         > extra-experimental-features = ${experimentalFeature}
         > ```
       '';
-    in result;
+    in
+    result;
 
-  storesList = map
-    (name: rec {
-      inherit name;
-      slug = replaceStrings [ " " ] [ "-" ] (toLower name);
-      filename = "${slug}.md";
-      page = showStore { inherit name slug; } storeInfo.${name};
-    })
-    (attrNames storeInfo);
+  storesList = map (name: rec {
+    inherit name;
+    slug = replaceStrings [ " " ] [ "-" ] (toLower name);
+    filename = "${slug}.md";
+    page = showStore { inherit name slug; } storeInfo.${name};
+  }) (attrNames storeInfo);
 
-in storesList
+in
+storesList

doc/manual/generate-store-types.nix

@@ -1,5 +1,11 @@
 let
-  inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings;
+  inherit (builtins)
+    attrNames
+    listToAttrs
+    concatStringsSep
+    readFile
+    replaceStrings
+    ;
   showSettings = import <nix/generate-settings.nix>;
   showStoreDocs = import <nix/generate-store-info.nix>;
 in
@@ -14,26 +20,28 @@ let
 
   index =
     let
-      showEntry = store:
-        "- [${store.name}](./${store.filename})";
+      showEntry = store: "- [${store.name}](./${store.filename})";
     in
     concatStringsSep "\n" (map showEntry storesList);
 
-  "index.md" = replaceStrings
-    [ "@store-types@" ] [ index ]
-    (readFile ./source/store/types/index.md.in);
+  "index.md" =
+    replaceStrings [ "@store-types@" ] [ index ]
+      (readFile ./source/store/types/index.md.in);
 
   tableOfContents =
     let
-      showEntry = store:
-        "    - [${store.name}](store/types/${store.filename})";
+      showEntry = store: "    - [${store.name}](store/types/${store.filename})";
     in
     concatStringsSep "\n" (map showEntry storesList) + "\n";
 
   "SUMMARY.md" = tableOfContents;
 
-  storePages = listToAttrs
-    (map (s: { name = s.filename; value = s.page; }) storesList);
+  storePages = listToAttrs (
+    map (s: {
+      name = s.filename;
+      value = s.page;
+    }) storesList
+  );
 
 in
 storePages // { inherit "index.md" "SUMMARY.md"; }

doc/manual/generate-xp-features-shortlist.nix

@@ -2,8 +2,8 @@ with builtins;
 with import <nix/utils.nix>;
 
 let
-  showExperimentalFeature = name: doc:
-    ''
-      - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name})
-    '';
-in xps: indent "  " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps)))
+  showExperimentalFeature = name: doc: ''
+    - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name})
+  '';
+in
+xps: indent "  " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps)))

doc/manual/generate-xp-features.nix

@@ -2,7 +2,8 @@ with builtins;
 with import <nix/utils.nix>;
 
 let
-  showExperimentalFeature = name: doc:
+  showExperimentalFeature =
+    name: doc:
     squash ''
       ## [`${name}`]{#xp-feature-${name}}
 

doc/manual/package.nix

@@ -1,19 +1,20 @@
-{ lib
-, mkMesonDerivation
+{
+  lib,
+  mkMesonDerivation,
 
-, meson
-, ninja
-, lowdown-unsandboxed
-, mdbook
-, mdbook-linkcheck
-, jq
-, python3
-, rsync
-, nix-cli
+  meson,
+  ninja,
+  lowdown-unsandboxed,
+  mdbook,
+  mdbook-linkcheck,
+  jq,
+  python3,
+  rsync,
+  nix-cli,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -25,18 +26,22 @@ mkMesonDerivation (finalAttrs: {
   inherit version;
 
   workDir = ./.;
-  fileset = fileset.difference
-    (fileset.unions [
-      ../../.version
-      # Too many different types of files to filter for now
-      ../../doc/manual
-      ./.
-    ])
-    # Do a blacklist instead
-    ../../doc/manual/package.nix;
+  fileset =
+    fileset.difference
+      (fileset.unions [
+        ../../.version
+        # Too many different types of files to filter for now
+        ../../doc/manual
+        ./.
+      ])
+      # Do a blacklist instead
+      ../../doc/manual/package.nix;
 
   # TODO the man pages should probably be separate
-  outputs = [ "out" "man" ];
+  outputs = [
+    "out"
+    "man"
+  ];
 
   # Hack for sake of the dev shell
   passthru.externalNativeBuildInputs = [
@@ -54,11 +59,10 @@ mkMesonDerivation (finalAttrs: {
     nix-cli
   ];
 
-  preConfigure =
-    ''
-      chmod u+w ./.version
-      echo ${finalAttrs.version} > ./.version
-    '';
+  preConfigure = ''
+    chmod u+w ./.version
+    echo ${finalAttrs.version} > ./.version
+  '';
 
   postInstall = ''
     mkdir -p ''$out/nix-support

doc/manual/rl-next/curl-cloexec.md

@@ -0,0 +1,10 @@
+---
+synopsis: Set FD_CLOEXEC on sockets created by curl
+issues: []
+prs: [12439]
+---
+
+
+Curl creates sockets without setting FD_CLOEXEC/SOCK_CLOEXEC, this can cause connections to remain open forever when using commands like `nix shell`
+
+This change sets the FD_CLOEXEC flag using a CURLOPT_SOCKOPTFUNCTION callback.

doc/manual/utils.nix

@@ -11,10 +11,15 @@ rec {
 
   concatStrings = concatStringsSep "";
 
-  attrsToList = a:
-    map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a);
+  attrsToList =
+    a:
+    map (name: {
+      inherit name;
+      value = a.${name};
+    }) (builtins.attrNames a);
 
-  replaceStringsRec = from: to: string:
+  replaceStringsRec =
+    from: to: string:
     # recursively replace occurrences of `from` with `to` within `string`
     # example:
     #     replaceStringRec "--" "-" "hello-----world"
@@ -22,16 +27,18 @@ rec {
     let
       replaced = replaceStrings [ from ] [ to ] string;
     in
-      if replaced == string then string else replaceStringsRec from to replaced;
+    if replaced == string then string else replaceStringsRec from to replaced;
 
   toLower = replaceStrings upperChars lowerChars;
 
   squash = replaceStringsRec "\n\n\n" "\n\n";
 
-  trim = string:
+  trim =
+    string:
     # trim trailing spaces and squash non-leading spaces
     let
-      trimLine = line:
+      trimLine =
+        line:
         let
           # separate leading spaces from the rest
           parts = split "(^ *)" line;
@@ -39,19 +46,30 @@ rec {
           rest = elemAt parts 2;
           # drop trailing spaces
           body = head (split " *$" rest);
-        in spaces + replaceStringsRec "  " " " body;
-    in concatStringsSep "\n" (map trimLine (splitLines string));
+        in
+        spaces + replaceStringsRec "  " " " body;
+    in
+    concatStringsSep "\n" (map trimLine (splitLines string));
 
   # FIXME: O(n^2)
-  unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) [];
+  unique = foldl' (acc: e: if elem e acc then acc else acc ++ [ e ]) [ ];
 
   nameValuePair = name: value: { inherit name value; };
 
-  filterAttrs = pred: set:
-    listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set));
+  filterAttrs =
+    pred: set:
+    listToAttrs (
+      concatMap (
+        name:
+        let
+          v = set.${name};
+        in
+        if pred name v then [ (nameValuePair name v) ] else [ ]
+      ) (attrNames set)
+    );
 
   optionalString = cond: string: if cond then string else "";
 
-  indent = prefix: s:
-    concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
+  indent =
+    prefix: s: concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s));
 }

docker.nix

@@ -1,112 +1,113 @@
-{ pkgs ? import <nixpkgs> { }
-, lib ? pkgs.lib
-, name ? "nix"
-, tag ? "latest"
-, bundleNixpkgs ? true
-, channelName ? "nixpkgs"
-, channelURL ? "https://nixos.org/channels/nixpkgs-unstable"
-, extraPkgs ? []
-, maxLayers ? 100
-, nixConf ? {}
-, flake-registry ? null
-, uid ? 0
-, gid ? 0
-, uname ? "root"
-, gname ? "root"
+{
+  pkgs ? import <nixpkgs> { },
+  lib ? pkgs.lib,
+  name ? "nix",
+  tag ? "latest",
+  bundleNixpkgs ? true,
+  channelName ? "nixpkgs",
+  channelURL ? "https://nixos.org/channels/nixpkgs-unstable",
+  extraPkgs ? [ ],
+  maxLayers ? 100,
+  nixConf ? { },
+  flake-registry ? null,
+  uid ? 0,
+  gid ? 0,
+  uname ? "root",
+  gname ? "root",
 }:
 let
-  defaultPkgs = with pkgs; [
-    nix
-    bashInteractive
-    coreutils-full
-    gnutar
-    gzip
-    gnugrep
-    which
-    curl
-    less
-    wget
-    man
-    cacert.out
-    findutils
-    iana-etc
-    git
-    openssh
-  ] ++ extraPkgs;
+  defaultPkgs =
+    with pkgs;
+    [
+      nix
+      bashInteractive
+      coreutils-full
+      gnutar
+      gzip
+      gnugrep
+      which
+      curl
+      less
+      wget
+      man
+      cacert.out
+      findutils
+      iana-etc
+      git
+      openssh
+    ]
+    ++ extraPkgs;
 
-  users = {
+  users =
+    {
 
-    root = {
-      uid = 0;
-      shell = "${pkgs.bashInteractive}/bin/bash";
-      home = "/root";
-      gid = 0;
-      groups = [ "root" ];
-      description = "System administrator";
+      root = {
+        uid = 0;
+        shell = "${pkgs.bashInteractive}/bin/bash";
+        home = "/root";
+        gid = 0;
+        groups = [ "root" ];
+        description = "System administrator";
+      };
+
+      nobody = {
+        uid = 65534;
+        shell = "${pkgs.shadow}/bin/nologin";
+        home = "/var/empty";
+        gid = 65534;
+        groups = [ "nobody" ];
+        description = "Unprivileged account (don't use!)";
+      };
+
+    }
+    // lib.optionalAttrs (uid != 0) {
+      "${uname}" = {
+        uid = uid;
+        shell = "${pkgs.bashInteractive}/bin/bash";
+        home = "/home/${uname}";
+        gid = gid;
+        groups = [ "${gname}" ];
+        description = "Nix user";
+      };
+    }
+    // lib.listToAttrs (
+      map (n: {
+        name = "nixbld${toString n}";
+        value = {
+          uid = 30000 + n;
+          gid = 30000;
+          groups = [ "nixbld" ];
+          description = "Nix build user ${toString n}";
+        };
+      }) (lib.lists.range 1 32)
+    );
+
+  groups =
+    {
+      root.gid = 0;
+      nixbld.gid = 30000;
+      nobody.gid = 65534;
+    }
+    // lib.optionalAttrs (gid != 0) {
+      "${gname}".gid = gid;
     };
 
-    nobody = {
-      uid = 65534;
-      shell = "${pkgs.shadow}/bin/nologin";
-      home = "/var/empty";
-      gid = 65534;
-      groups = [ "nobody" ];
-      description = "Unprivileged account (don't use!)";
-    };
-
-  } // lib.optionalAttrs (uid != 0) {
-    "${uname}" = {
-      uid = uid;
-      shell = "${pkgs.bashInteractive}/bin/bash";
-      home = "/home/${uname}";
-      gid = gid;
-      groups = [ "${gname}" ];
-      description = "Nix user";
-    };
-  } // lib.listToAttrs (
-    map
-      (
-        n: {
-          name = "nixbld${toString n}";
-          value = {
-            uid = 30000 + n;
-            gid = 30000;
-            groups = [ "nixbld" ];
-            description = "Nix build user ${toString n}";
-          };
-        }
-      )
-      (lib.lists.range 1 32)
-  );
-
-  groups = {
-    root.gid = 0;
-    nixbld.gid = 30000;
-    nobody.gid = 65534;
-  } // lib.optionalAttrs (gid != 0) {
-    "${gname}".gid = gid;
-  };
-
   userToPasswd = (
     k:
-    { uid
-    , gid ? 65534
-    , home ? "/var/empty"
-    , description ? ""
-    , shell ? "/bin/false"
-    , groups ? [ ]
-    }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}"
-  );
-  passwdContents = (
-    lib.concatStringsSep "\n"
-      (lib.attrValues (lib.mapAttrs userToPasswd users))
+    {
+      uid,
+      gid ? 65534,
+      home ? "/var/empty",
+      description ? "",
+      shell ? "/bin/false",
+      groups ? [ ],
+    }:
+    "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}"
   );
+  passwdContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs userToPasswd users)));
 
   userToShadow = k: { ... }: "${k}:!:1::::::";
-  shadowContents = (
-    lib.concatStringsSep "\n"
-      (lib.attrValues (lib.mapAttrs userToShadow users))
-  );
+  shadowContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs userToShadow users)));
 
   # Map groups to members
   # {
@@ -116,42 +117,35 @@ let
     let
       # Create a flat list of user/group mappings
       mappings = (
-        builtins.foldl'
-          (
-            acc: user:
-              let
-                groups = users.${user}.groups or [ ];
-              in
-              acc ++ map
-                (group: {
-                  inherit user group;
-                })
-                groups
-          )
-          [ ]
-          (lib.attrNames users)
+        builtins.foldl' (
+          acc: user:
+          let
+            groups = users.${user}.groups or [ ];
+          in
+          acc
+          ++ map (group: {
+            inherit user group;
+          }) groups
+        ) [ ] (lib.attrNames users)
       );
     in
-    (
-      builtins.foldl'
-        (
-          acc: v: acc // {
-            ${v.group} = acc.${v.group} or [ ] ++ [ v.user ];
-          }
-        )
-        { }
-        mappings)
+    (builtins.foldl' (
+      acc: v:
+      acc
+      // {
+        ${v.group} = acc.${v.group} or [ ] ++ [ v.user ];
+      }
+    ) { } mappings)
   );
 
-  groupToGroup = k: { gid }:
+  groupToGroup =
+    k:
+    { gid }:
     let
       members = groupMemberMap.${k} or [ ];
     in
     "${k}:x:${toString gid}:${lib.concatStringsSep "," members}";
-  groupContents = (
-    lib.concatStringsSep "\n"
-      (lib.attrValues (lib.mapAttrs groupToGroup groups))
-  );
+  groupContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs groupToGroup groups)));
 
   defaultNixConf = {
     sandbox = "false";
@@ -159,11 +153,17 @@ let
     trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ];
   };
 
-  nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v:
-    let
-      vStr = if builtins.isList v then lib.concatStringsSep " " v else v;
-    in
-      "${n} = ${vStr}") (defaultNixConf // nixConf))) + "\n";
+  nixConfContents =
+    (lib.concatStringsSep "\n" (
+      lib.mapAttrsFlatten (
+        n: v:
+        let
+          vStr = if builtins.isList v then lib.concatStringsSep " " v else v;
+        in
+        "${n} = ${vStr}"
+      ) (defaultNixConf // nixConf)
+    ))
+    + "\n";
 
   userHome = if uid == 0 then "/root" else "/home/${uname}";
 
@@ -184,21 +184,29 @@ let
       manifest = pkgs.buildPackages.runCommand "manifest.nix" { } ''
         cat > $out <<EOF
         [
-        ${lib.concatStringsSep "\n" (builtins.map (drv: let
-          outputs = drv.outputsToInstall or [ "out" ];
-        in ''
-          {
-            ${lib.concatStringsSep "\n" (builtins.map (output: ''
-              ${output} = { outPath = "${lib.getOutput output drv}"; };
-            '') outputs)}
-            outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
-            name = "${drv.name}";
-            outPath = "${drv}";
-            system = "${drv.system}";
-            type = "derivation";
-            meta = { };
-          }
-        '') defaultPkgs)}
+        ${lib.concatStringsSep "\n" (
+          builtins.map (
+            drv:
+            let
+              outputs = drv.outputsToInstall or [ "out" ];
+            in
+            ''
+              {
+                ${lib.concatStringsSep "\n" (
+                  builtins.map (output: ''
+                    ${output} = { outPath = "${lib.getOutput output drv}"; };
+                  '') outputs
+                )}
+                outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ];
+                name = "${drv.name}";
+                outPath = "${drv}";
+                system = "${drv.system}";
+                type = "derivation";
+                meta = { };
+              }
+            ''
+          ) defaultPkgs
+        )}
         ]
         EOF
       '';
@@ -207,16 +215,22 @@ let
         cp -a ${rootEnv}/* $out/
         ln -s ${manifest} $out/manifest.nix
       '';
-      flake-registry-path = if (flake-registry == null) then
-        null
-      else if (builtins.readFileType (toString flake-registry)) == "directory" then
-        "${flake-registry}/flake-registry.json"
-      else
-        flake-registry;
+      flake-registry-path =
+        if (flake-registry == null) then
+          null
+        else if (builtins.readFileType (toString flake-registry)) == "directory" then
+          "${flake-registry}/flake-registry.json"
+        else
+          flake-registry;
     in
     pkgs.runCommand "base-system"
       {
-        inherit passwdContents groupContents shadowContents nixConfContents;
+        inherit
+          passwdContents
+          groupContents
+          shadowContents
+          nixConfContents
+          ;
         passAsFile = [
           "passwdContents"
           "groupContents"
@@ -225,67 +239,79 @@ let
         ];
         allowSubstitutes = false;
         preferLocalBuild = true;
-      } (''
-      env
-      set -x
-      mkdir -p $out/etc
+      }
+      (
+        ''
+          env
+          set -x
+          mkdir -p $out/etc
 
-      mkdir -p $out/etc/ssl/certs
-      ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
+          mkdir -p $out/etc/ssl/certs
+          ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs
 
-      cat $passwdContentsPath > $out/etc/passwd
-      echo "" >> $out/etc/passwd
+          cat $passwdContentsPath > $out/etc/passwd
+          echo "" >> $out/etc/passwd
 
-      cat $groupContentsPath > $out/etc/group
-      echo "" >> $out/etc/group
+          cat $groupContentsPath > $out/etc/group
+          echo "" >> $out/etc/group
 
-      cat $shadowContentsPath > $out/etc/shadow
-      echo "" >> $out/etc/shadow
+          cat $shadowContentsPath > $out/etc/shadow
+          echo "" >> $out/etc/shadow
 
-      mkdir -p $out/usr
-      ln -s /nix/var/nix/profiles/share $out/usr/
+          mkdir -p $out/usr
+          ln -s /nix/var/nix/profiles/share $out/usr/
 
-      mkdir -p $out/nix/var/nix/gcroots
+          mkdir -p $out/nix/var/nix/gcroots
 
-      mkdir $out/tmp
+          mkdir $out/tmp
 
-      mkdir -p $out/var/tmp
+          mkdir -p $out/var/tmp
 
-      mkdir -p $out/etc/nix
-      cat $nixConfContentsPath > $out/etc/nix/nix.conf
+          mkdir -p $out/etc/nix
+          cat $nixConfContentsPath > $out/etc/nix/nix.conf
 
-      mkdir -p $out${userHome}
-      mkdir -p $out/nix/var/nix/profiles/per-user/${uname}
+          mkdir -p $out${userHome}
+          mkdir -p $out/nix/var/nix/profiles/per-user/${uname}
 
-      ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
-      ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
-      ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile
+          ln -s ${profile} $out/nix/var/nix/profiles/default-1-link
+          ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default
+          ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile
 
-      ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link
-      ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels
+          ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link
+          ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels
 
-      mkdir -p $out${userHome}/.nix-defexpr
-      ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels
-      echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels
+          mkdir -p $out${userHome}/.nix-defexpr
+          ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels
+          echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels
 
-      mkdir -p $out/bin $out/usr/bin
-      ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
-      ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
+          mkdir -p $out/bin $out/usr/bin
+          ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env
+          ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh
 
-    '' + (lib.optionalString (flake-registry-path != null) ''
-      nixCacheDir="${userHome}/.cache/nix"
-      mkdir -p $out$nixCacheDir
-      globalFlakeRegistryPath="$nixCacheDir/flake-registry.json"
-      ln -s ${flake-registry-path} $out$globalFlakeRegistryPath
-      mkdir -p $out/nix/var/nix/gcroots/auto
-      rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath))
-      ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName
-    ''));
+        ''
+        + (lib.optionalString (flake-registry-path != null) ''
+          nixCacheDir="${userHome}/.cache/nix"
+          mkdir -p $out$nixCacheDir
+          globalFlakeRegistryPath="$nixCacheDir/flake-registry.json"
+          ln -s ${flake-registry-path} $out$globalFlakeRegistryPath
+          mkdir -p $out/nix/var/nix/gcroots/auto
+          rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath))
+          ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName
+        '')
+      );
 
 in
 pkgs.dockerTools.buildLayeredImageWithNixDb {
 
-  inherit name tag maxLayers uid gid uname gname;
+  inherit
+    name
+    tag
+    maxLayers
+    uid
+    gid
+    uname
+    gname
+    ;
 
   contents = [ baseSystem ];
 
@@ -305,15 +331,19 @@ pkgs.dockerTools.buildLayeredImageWithNixDb {
     User = "${toString uid}:${toString gid}";
     Env = [
       "USER=${uname}"
-      "PATH=${lib.concatStringsSep ":" [
-        "${userHome}/.nix-profile/bin"
-        "/nix/var/nix/profiles/default/bin"
-        "/nix/var/nix/profiles/default/sbin"
-      ]}"
-      "MANPATH=${lib.concatStringsSep ":" [
-        "${userHome}/.nix-profile/share/man"
-        "/nix/var/nix/profiles/default/share/man"
-      ]}"
+      "PATH=${
+        lib.concatStringsSep ":" [
+          "${userHome}/.nix-profile/bin"
+          "/nix/var/nix/profiles/default/bin"
+          "/nix/var/nix/profiles/default/sbin"
+        ]
+      }"
+      "MANPATH=${
+        lib.concatStringsSep ":" [
+          "${userHome}/.nix-profile/share/man"
+          "/nix/var/nix/profiles/default/share/man"
+        ]
+      }"
       "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
       "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
       "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"

flake.nix

@@ -5,7 +5,10 @@
 
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
-  inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
+  inputs.flake-compat = {
+    url = "github:edolstra/flake-compat";
+    flake = false;
+  };
 
   # dev tooling
   inputs.flake-parts.url = "github:hercules-ci/flake-parts";
@@ -18,18 +21,29 @@
   inputs.git-hooks-nix.inputs.flake-compat.follows = "";
   inputs.git-hooks-nix.inputs.gitignore.follows = "";
 
-  outputs = inputs@{ self, nixpkgs, nixpkgs-regression, ... }:
-
+  outputs =
+    inputs@{
+      self,
+      nixpkgs,
+      nixpkgs-regression,
+      ...
+    }:
 
     let
       inherit (nixpkgs) lib;
 
-      officialRelease = false;
+      officialRelease = true;
 
       linux32BitSystems = [ "i686-linux" ];
-      linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ];
+      linux64BitSystems = [
+        "x86_64-linux"
+        "aarch64-linux"
+      ];
       linuxSystems = linux32BitSystems ++ linux64BitSystems;
-      darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ];
+      darwinSystems = [
+        "x86_64-darwin"
+        "aarch64-darwin"
+      ];
       systems = linuxSystems ++ darwinSystems;
 
       crossSystems = [
@@ -59,63 +73,77 @@
         (Provided that the names are unique.)
 
         See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs
-       */
+      */
       flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs;
 
       forAllSystems = lib.genAttrs systems;
 
       forAllCrossSystems = lib.genAttrs crossSystems;
 
-      forAllStdenvs = f:
-        lib.listToAttrs
-          (map
-            (stdenvName: {
-              name = "${stdenvName}Packages";
-              value = f stdenvName;
-            })
-            stdenvs);
-
+      forAllStdenvs =
+        f:
+        lib.listToAttrs (
+          map (stdenvName: {
+            name = "${stdenvName}Packages";
+            value = f stdenvName;
+          }) stdenvs
+        );
 
       # We don't apply flake-parts to the whole flake so that non-development attributes
       # load without fetching any development inputs.
       devFlake = inputs.flake-parts.lib.mkFlake { inherit inputs; } {
         imports = [ ./maintainers/flake-module.nix ];
         systems = lib.subtractLists crossSystems systems;
-        perSystem = { system, ... }: {
-          _module.args.pkgs = nixpkgsFor.${system}.native;
-        };
+        perSystem =
+          { system, ... }:
+          {
+            _module.args.pkgs = nixpkgsFor.${system}.native;
+          };
       };
 
       # Memoize nixpkgs for different platforms for efficiency.
-      nixpkgsFor = forAllSystems
-        (system: let
-          make-pkgs = crossSystem: stdenv: import nixpkgs {
-            localSystem = {
-              inherit system;
+      nixpkgsFor = forAllSystems (
+        system:
+        let
+          make-pkgs =
+            crossSystem: stdenv:
+            import nixpkgs {
+              localSystem = {
+                inherit system;
+              };
+              crossSystem =
+                if crossSystem == null then
+                  null
+                else
+                  {
+                    config = crossSystem;
+                  }
+                  // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
+                    useLLVM = true;
+                  };
+              overlays = [
+                (overlayFor (p: p.${stdenv}))
+              ];
             };
-            crossSystem = if crossSystem == null then null else {
-              config = crossSystem;
-            } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") {
-              useLLVM = true;
-            };
-            overlays = [
-              (overlayFor (p: p.${stdenv}))
-            ];
-          };
           stdenvs = forAllStdenvs (make-pkgs null);
           native = stdenvs.stdenvPackages;
-        in {
+        in
+        {
           inherit stdenvs native;
           static = native.pkgsStatic;
           llvm = native.pkgsLLVM;
           cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv");
-        });
+        }
+      );
 
-      binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix {
-        inherit nix;
-      };
+      binaryTarball =
+        nix: pkgs:
+        pkgs.callPackage ./scripts/binary-tarball.nix {
+          inherit nix;
+        };
 
-      overlayFor = getStdenv: final: prev:
+      overlayFor =
+        getStdenv: final: prev:
         let
           stdenv = getStdenv final;
         in
@@ -137,7 +165,9 @@
                 f = import ./packaging/components.nix {
                   inherit (final) lib;
                   inherit officialRelease;
+                  pkgs = final;
                   src = self;
+                  maintainers = [ ];
                 };
               };
 
@@ -162,12 +192,19 @@
           # See https://github.com/NixOS/nixpkgs/pull/214409
           # Remove when fixed in this flake's nixpkgs
           pre-commit =
-            if prev.stdenv.hostPlatform.system == "i686-linux"
-            then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; })
-            else prev.pre-commit;
+            if prev.stdenv.hostPlatform.system == "i686-linux" then
+              (prev.pre-commit.override (o: {
+                dotnet-sdk = "";
+              })).overridePythonAttrs
+                (o: {
+                  doCheck = false;
+                })
+            else
+              prev.pre-commit;
         };
 
-    in {
+    in
+    {
       # A Nixpkgs overlay that overrides the 'nix' and
       # 'nix-perl-bindings' packages.
       overlays.default = overlayFor (p: p.stdenv);
@@ -186,53 +223,93 @@
           ;
       };
 
-      checks = forAllSystems (system: {
-        installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
-        installTests = self.hydraJobs.installTests.${system};
-        nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
-        rl-next =
-          let pkgs = nixpkgsFor.${system}.native;
-          in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
-          LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
-        '';
-        repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { };
-      } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
-        dockerImage = self.hydraJobs.dockerImage.${system};
-      } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {
-        # Some perl dependencies are broken on i686-linux.
-        # Since the support is only best-effort there, disable the perl
-        # bindings
-        perlBindings = self.hydraJobs.perlBindings.${system};
-      }
-      # Add "passthru" tests
-      // flatMapAttrs ({
-          "" = nixpkgsFor.${system}.native;
-        } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) {
-          # TODO: enable static builds for darwin, blocked on:
-          #       https://github.com/NixOS/nixpkgs/issues/320448
-          # TODO: disabled to speed up GHA CI.
-          #"static-" = nixpkgsFor.${system}.static;
-        })
-        (nixpkgsPrefix: nixpkgs:
-          flatMapAttrs nixpkgs.nixComponents
-            (pkgName: pkg:
-              flatMapAttrs pkg.tests or {}
-              (testName: test: {
-                "${nixpkgsPrefix}${pkgName}-${testName}" = test;
-              })
+      checks = forAllSystems (
+        system:
+        {
+          installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
+          installTests = self.hydraJobs.installTests.${system};
+          nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system};
+          rl-next =
+            let
+              pkgs = nixpkgsFor.${system}.native;
+            in
+            pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } ''
+              LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out
+            '';
+          repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { };
+
+          /**
+            Checks for our packaging expressions.
+            This shouldn't build anything significant; just check that things
+            (including derivations) are _set up_ correctly.
+          */
+          packaging-overriding =
+            let
+              pkgs = nixpkgsFor.${system}.native;
+              nix = self.packages.${system}.nix;
+            in
+            assert (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src.patches == [ pkgs.emptyFile ];
+            if pkgs.stdenv.buildPlatform.isDarwin then
+              lib.warn "packaging-overriding check currently disabled because of a permissions issue on macOS" pkgs.emptyFile
+            else
+              # If this fails, something might be wrong with how we've wired the scope,
+              # or something could be broken in Nixpkgs.
+              pkgs.testers.testEqualContents {
+                assertion = "trivial patch does not change source contents";
+                expected = "${./.}";
+                actual =
+                  # Same for all components; nix-util is an arbitrary pick
+                  (nix.appendPatches [ pkgs.emptyFile ]).libs.nix-util.src;
+              };
+        }
+        // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) {
+          dockerImage = self.hydraJobs.dockerImage.${system};
+        }
+        // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) {
+          # Some perl dependencies are broken on i686-linux.
+          # Since the support is only best-effort there, disable the perl
+          # bindings
+          perlBindings = self.hydraJobs.perlBindings.${system};
+        }
+        # Add "passthru" tests
+        //
+          flatMapAttrs
+            (
+              {
+                "" = nixpkgsFor.${system}.native;
+              }
+              // lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) {
+                # TODO: enable static builds for darwin, blocked on:
+                #       https://github.com/NixOS/nixpkgs/issues/320448
+                # TODO: disabled to speed up GHA CI.
+                #"static-" = nixpkgsFor.${system}.static;
+              }
             )
-          // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) {
-            "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests;
-          }
-        )
-      // devFlake.checks.${system} or {}
+            (
+              nixpkgsPrefix: nixpkgs:
+              flatMapAttrs nixpkgs.nixComponents (
+                pkgName: pkg:
+                flatMapAttrs pkg.tests or { } (
+                  testName: test: {
+                    "${nixpkgsPrefix}${pkgName}-${testName}" = test;
+                  }
+                )
+              )
+              // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) {
+                "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests;
+              }
+            )
+        // devFlake.checks.${system} or { }
       );
 
-      packages = forAllSystems (system:
-        { # Here we put attributes that map 1:1 into packages.<system>, ie
+      packages = forAllSystems (
+        system:
+        {
+          # Here we put attributes that map 1:1 into packages.<system>, ie
           # for which we don't apply the full build matrix such as cross or static.
           inherit (nixpkgsFor.${system}.native)
-            changelog-d;
+            changelog-d
+            ;
           default = self.packages.${system}.nix;
           installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system};
           binaryTarball = self.hydraJobs.binaryTarball.${system};
@@ -243,96 +320,143 @@
           nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs;
         }
         # We need to flatten recursive attribute sets of derivations to pass `flake check`.
-        // flatMapAttrs
-          { # Components we'll iterate over in the upcoming lambda
-            "nix-util" = { };
-            "nix-util-c" = { };
-            "nix-util-test-support" = { };
-            "nix-util-tests" = { };
+        //
+          flatMapAttrs
+            {
+              # Components we'll iterate over in the upcoming lambda
+              "nix-util" = { };
+              "nix-util-c" = { };
+              "nix-util-test-support" = { };
+              "nix-util-tests" = { };
 
-            "nix-store" = { };
-            "nix-store-c" = { };
-            "nix-store-test-support" = { };
-            "nix-store-tests" = { };
+              "nix-store" = { };
+              "nix-store-c" = { };
+              "nix-store-test-support" = { };
+              "nix-store-tests" = { };
 
-            "nix-fetchers" = { };
-            "nix-fetchers-tests" = { };
+              "nix-fetchers" = { };
+              "nix-fetchers-tests" = { };
 
-            "nix-expr" = { };
-            "nix-expr-c" = { };
-            "nix-expr-test-support" = { };
-            "nix-expr-tests" = { };
+              "nix-expr" = { };
+              "nix-expr-c" = { };
+              "nix-expr-test-support" = { };
+              "nix-expr-tests" = { };
 
-            "nix-flake" = { };
-            "nix-flake-tests" = { };
+              "nix-flake" = { };
+              "nix-flake-tests" = { };
 
-            "nix-main" = { };
-            "nix-main-c" = { };
+              "nix-main" = { };
+              "nix-main-c" = { };
 
-            "nix-cmd" = { };
+              "nix-cmd" = { };
 
-            "nix-cli" = { };
+              "nix-cli" = { };
 
-            "nix-everything" = { };
+              "nix-everything" = { };
 
-            "nix-functional-tests" = { supportsCross = false; };
+              "nix-functional-tests" = {
+                supportsCross = false;
+              };
 
-            "nix-perl-bindings" = { supportsCross = false; };
-          }
-          (pkgName: { supportsCross ? true }: {
-              # These attributes go right into `packages.<system>`.
-              "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
-              "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
-              "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName};
+              "nix-perl-bindings" = {
+                supportsCross = false;
+              };
             }
-            // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: {
-              # These attributes go right into `packages.<system>`.
-              "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
-            }))
-            // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: {
-              # These attributes go right into `packages.<system>`.
-              "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
-            })
-          )
+            (
+              pkgName:
+              {
+                supportsCross ? true,
+              }:
+              {
+                # These attributes go right into `packages.<system>`.
+                "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName};
+                "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName};
+                "${pkgName}-llvm" = nixpkgsFor.${system}.llvm.nixComponents.${pkgName};
+              }
+              // lib.optionalAttrs supportsCross (
+                flatMapAttrs (lib.genAttrs crossSystems (_: { })) (
+                  crossSystem:
+                  { }:
+                  {
+                    # These attributes go right into `packages.<system>`.
+                    "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName};
+                  }
+                )
+              )
+              // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (
+                stdenvName:
+                { }:
+                {
+                  # These attributes go right into `packages.<system>`.
+                  "${pkgName}-${stdenvName}" =
+                    nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName};
+                }
+              )
+            )
         // lib.optionalAttrs (builtins.elem system linux64BitSystems) {
-        dockerImage =
-          let
-            pkgs = nixpkgsFor.${system}.native;
-            image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; };
-          in
-          pkgs.runCommand
-            "docker-image-tarball-${pkgs.nix.version}"
-            { meta.description = "Docker image with Nix for ${system}"; }
-            ''
-              mkdir -p $out/nix-support
-              image=$out/image.tar.gz
-              ln -s ${image} $image
-              echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
-            '';
-      });
+          dockerImage =
+            let
+              pkgs = nixpkgsFor.${system}.native;
+              image = import ./docker.nix {
+                inherit pkgs;
+                tag = pkgs.nix.version;
+              };
+            in
+            pkgs.runCommand "docker-image-tarball-${pkgs.nix.version}"
+              { meta.description = "Docker image with Nix for ${system}"; }
+              ''
+                mkdir -p $out/nix-support
+                image=$out/image.tar.gz
+                ln -s ${image} $image
+                echo "file binary-dist $image" >> $out/nix-support/hydra-build-products
+              '';
+        }
+      );
 
-      devShells = let
-        makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; };
-        prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; });
-      in
-        forAllSystems (system:
-          prefixAttrs "native" (forAllStdenvs (stdenvName: makeShell {
-            pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages";
-          })) //
-          lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) (
-            prefixAttrs "static" (forAllStdenvs (stdenvName: makeShell {
-              pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic;
-            })) //
-            prefixAttrs "llvm" (forAllStdenvs (stdenvName: makeShell {
-              pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsLLVM;
-            })) //
-            prefixAttrs "cross" (forAllCrossSystems (crossSystem: makeShell {
-              pkgs = nixpkgsFor.${system}.cross.${crossSystem};
-            }))
-          ) //
-          {
+      devShells =
+        let
+          makeShell = import ./packaging/dev-shell.nix { inherit lib devFlake; };
+          prefixAttrs = prefix: lib.concatMapAttrs (k: v: { "${prefix}-${k}" = v; });
+        in
+        forAllSystems (
+          system:
+          prefixAttrs "native" (
+            forAllStdenvs (
+              stdenvName:
+              makeShell {
+                pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages";
+              }
+            )
+          )
+          // lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) (
+            prefixAttrs "static" (
+              forAllStdenvs (
+                stdenvName:
+                makeShell {
+                  pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsStatic;
+                }
+              )
+            )
+            // prefixAttrs "llvm" (
+              forAllStdenvs (
+                stdenvName:
+                makeShell {
+                  pkgs = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".pkgsLLVM;
+                }
+              )
+            )
+            // prefixAttrs "cross" (
+              forAllCrossSystems (
+                crossSystem:
+                makeShell {
+                  pkgs = nixpkgsFor.${system}.cross.${crossSystem};
+                }
+              )
+            )
+          )
+          // {
             default = self.devShells.${system}.native-stdenvPackages;
           }
         );
-  };
+    };
 }

maintainers/release-notes

@@ -2,6 +2,8 @@
 # vim: set filetype=bash:
 #!nix shell .#changelog-d --command bash
 
+set -euo pipefail
+
 # --- CONFIGURATION ---
 
 # This does double duty for

packaging/components.nix

@@ -1,29 +1,333 @@
 {
   lib,
+  pkgs,
   src,
   officialRelease,
+  maintainers,
 }:
 
 scope:
 
 let
-  inherit (scope) callPackage;
+  inherit (scope)
+    callPackage
+    ;
+  inherit
+    (scope.callPackage (
+      { stdenv }:
+      {
+        inherit stdenv;
+      }
+    ) { })
+    stdenv
+    ;
+  inherit (pkgs.buildPackages)
+    meson
+    ninja
+    pkg-config
+    ;
 
   baseVersion = lib.fileContents ../.version;
 
   versionSuffix = lib.optionalString (!officialRelease) "pre";
 
-  fineVersionSuffix = lib.optionalString
-    (!officialRelease)
-    "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}";
+  fineVersionSuffix =
+    lib.optionalString (!officialRelease)
+      "pre${
+        builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")
+      }_${src.shortRev or "dirty"}";
 
   fineVersion = baseVersion + fineVersionSuffix;
+
+  root = ../.;
+
+  # Indirection for Nixpkgs to override when package.nix files are vendored
+  filesetToSource = lib.fileset.toSource;
+
+  /**
+    Given a set of layers, create a mkDerivation-like function
+  */
+  mkPackageBuilder =
+    exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn);
+
+  setVersionLayer = finalAttrs: prevAttrs: {
+    preConfigure =
+      prevAttrs.preConfigure or ""
+      +
+        # Update the repo-global .version file.
+        # Symlink ./.version points there, but by default only workDir is writable.
+        ''
+          chmod u+w ./.version
+          echo ${finalAttrs.version} > ./.version
+        '';
+  };
+
+  localSourceLayer =
+    finalAttrs: prevAttrs:
+    let
+      workDirPath =
+        # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has
+        # the requirement that everything except passthru and meta must be
+        # serialized by mkDerivation, which doesn't work for this.
+        prevAttrs.workDir;
+
+      workDirSubpath = lib.path.removePrefix root workDirPath;
+      sources =
+        assert prevAttrs.fileset._type == "fileset";
+        prevAttrs.fileset;
+      src = lib.fileset.toSource {
+        fileset = sources;
+        inherit root;
+      };
+
+    in
+    {
+      sourceRoot = "${src.name}/" + workDirSubpath;
+      inherit src;
+
+      # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir.
+      fileset = null;
+      workDir = null;
+    };
+
+  resolveRelPath = p: lib.path.removePrefix root p;
+
+  makeFetchedSourceLayer =
+    finalScope: finalAttrs: prevAttrs:
+    let
+      workDirPath =
+        # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has
+        # the requirement that everything except passthru and meta must be
+        # serialized by mkDerivation, which doesn't work for this.
+        prevAttrs.workDir;
+
+      workDirSubpath = resolveRelPath workDirPath;
+
+    in
+    {
+      sourceRoot = "${finalScope.patchedSrc.name}/" + workDirSubpath;
+      src = finalScope.patchedSrc;
+      version =
+        let
+          n = lib.length finalScope.patches;
+        in
+        if n == 0 then prevAttrs.version else prevAttrs.version + "+${toString n}";
+
+      # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir.
+      fileset = null;
+      workDir = null;
+    };
+
+  mesonLayer = finalAttrs: prevAttrs: {
+    # NOTE:
+    # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26,
+    # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default.
+    # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype.
+    mesonBuildType = "release";
+    # NOTE:
+    # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the
+    # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10.
+    # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable.
+    preConfigure =
+      prevAttrs.preConfigure or ""
+      +
+        lib.optionalString
+          (
+            !stdenv.hostPlatform.isWindows
+            # build failure
+            && !stdenv.hostPlatform.isStatic
+            # LTO breaks exception handling on x86-64-darwin.
+            && stdenv.system != "x86_64-darwin"
+          )
+          ''
+            case "$mesonBuildType" in
+            release|minsize) appendToVar mesonFlags "-Db_lto=true"  ;;
+            *)               appendToVar mesonFlags "-Db_lto=false" ;;
+            esac
+          '';
+    nativeBuildInputs = [
+      meson
+      ninja
+    ] ++ prevAttrs.nativeBuildInputs or [ ];
+    mesonCheckFlags = prevAttrs.mesonCheckFlags or [ ] ++ [
+      "--print-errorlogs"
+    ];
+  };
+
+  mesonBuildLayer = finalAttrs: prevAttrs: {
+    nativeBuildInputs = prevAttrs.nativeBuildInputs or [ ] ++ [
+      pkg-config
+    ];
+    separateDebugInfo = !stdenv.hostPlatform.isStatic;
+    hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
+    env =
+      prevAttrs.env or { }
+      // lib.optionalAttrs (
+        stdenv.isLinux
+        && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")
+        && !(stdenv.hostPlatform.useLLVM or false)
+      ) { LDFLAGS = "-fuse-ld=gold"; };
+  };
+
+  mesonLibraryLayer = finalAttrs: prevAttrs: {
+    outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
+  };
+
+  # Work around weird `--as-needed` linker behavior with BSD, see
+  # https://github.com/mesonbuild/meson/issues/3593
+  bsdNoLinkAsNeeded =
+    finalAttrs: prevAttrs:
+    lib.optionalAttrs stdenv.hostPlatform.isBSD {
+      mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or [ ];
+    };
+
+  nixDefaultsLayer = finalAttrs: prevAttrs: {
+    strictDeps = prevAttrs.strictDeps or true;
+    enableParallelBuilding = true;
+    pos = builtins.unsafeGetAttrPos "pname" prevAttrs;
+    meta = prevAttrs.meta or { } // {
+      homepage = prevAttrs.meta.homepage or "https://nixos.org/nix";
+      longDescription =
+        prevAttrs.longDescription or ''
+          Nix is a powerful package manager for mainly Linux and other Unix systems that
+          makes package management reliable and reproducible. It provides atomic
+          upgrades and rollbacks, side-by-side installation of multiple versions of
+          a package, multi-user package management and easy setup of build
+          environments.
+        '';
+      license = prevAttrs.meta.license or lib.licenses.lgpl21Plus;
+      maintainers = prevAttrs.meta.maintainers or [ ] ++ scope.maintainers;
+      platforms = prevAttrs.meta.platforms or (lib.platforms.unix ++ lib.platforms.windows);
+    };
+  };
+
+  /**
+    Append patches to the source layer.
+  */
+  appendPatches =
+    scope: patches:
+    scope.overrideScope (
+      finalScope: prevScope: {
+        patches = prevScope.patches ++ patches;
+      }
+    );
+
 in
 
 # This becomes the pkgs.nixComponents attribute set
 {
   version = baseVersion + versionSuffix;
   inherit versionSuffix;
+  inherit maintainers;
+
+  inherit filesetToSource;
+
+  /**
+    A user-provided extension function to apply to each component derivation.
+  */
+  mesonComponentOverrides = finalAttrs: prevAttrs: { };
+
+  /**
+    An overridable derivation layer for handling the sources.
+  */
+  sourceLayer = localSourceLayer;
+
+  /**
+    Resolve a path value to either itself or a path in the `src`, depending
+    whether `overrideSource` was called.
+  */
+  resolvePath = p: p;
+
+  /**
+    Apply an extension function (i.e. overlay-shaped) to all component derivations.
+  */
+  overrideAllMesonComponents =
+    f:
+    scope.overrideScope (
+      finalScope: prevScope: {
+        mesonComponentOverrides = lib.composeExtensions scope.mesonComponentOverrides f;
+      }
+    );
+
+  /**
+    Provide an alternate source. This allows the expressions to be vendored without copying the sources,
+    but it does make the build non-granular; all components will use a complete source.
+
+    Packaging expressions will be ignored.
+
+    Single argument: the source to use.
+
+    See also `appendPatches`
+  */
+  overrideSource =
+    src:
+    scope.overrideScope (
+      finalScope: prevScope: {
+        sourceLayer = makeFetchedSourceLayer finalScope;
+        /**
+          Unpatched source for the build of Nix. Packaging expressions will be ignored.
+        */
+        src = src;
+        /**
+          Patches for the whole Nix source. Changes to packaging expressions will be ignored.
+        */
+        patches = [ ];
+        /**
+          Fetched and patched source to be used in component derivations.
+        */
+        patchedSrc =
+          if finalScope.patches == [ ] then
+            src
+          else
+            pkgs.buildPackages.srcOnly (
+              pkgs.buildPackages.stdenvNoCC.mkDerivation {
+                name = "${finalScope.src.name or "nix-source"}-patched";
+                inherit (finalScope) src patches;
+              }
+            );
+        resolvePath = p: finalScope.patchedSrc + "/${resolveRelPath p}";
+        filesetToSource = { root, fileset }: finalScope.resolvePath root;
+        appendPatches = appendPatches finalScope;
+      }
+    );
+
+  /**
+    Append patches to be applied to the whole Nix source.
+    This affects all components.
+
+    Changes to the packaging expressions will be ignored.
+  */
+  appendPatches =
+    patches:
+    # switch to "fetched" source first, so that patches apply to the whole tree.
+    (scope.overrideSource "${./..}").appendPatches patches;
+
+  mkMesonDerivation = mkPackageBuilder [
+    nixDefaultsLayer
+    scope.sourceLayer
+    setVersionLayer
+    mesonLayer
+    scope.mesonComponentOverrides
+  ];
+  mkMesonExecutable = mkPackageBuilder [
+    nixDefaultsLayer
+    bsdNoLinkAsNeeded
+    scope.sourceLayer
+    setVersionLayer
+    mesonLayer
+    mesonBuildLayer
+    scope.mesonComponentOverrides
+  ];
+  mkMesonLibrary = mkPackageBuilder [
+    nixDefaultsLayer
+    bsdNoLinkAsNeeded
+    scope.sourceLayer
+    mesonLayer
+    setVersionLayer
+    mesonBuildLayer
+    mesonLibraryLayer
+    scope.mesonComponentOverrides
+  ];
 
   nix-util = callPackage ../src/libutil/package.nix { };
   nix-util-c = callPackage ../src/libutil-c/package.nix { };
@@ -54,7 +358,9 @@ in
 
   nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; };
 
-  nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; };
+  nix-functional-tests = callPackage ../tests/functional/package.nix {
+    version = fineVersion;
+  };
 
   nix-manual = callPackage ../doc/manual/package.nix { version = fineVersion; };
   nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; };
@@ -62,5 +368,57 @@ in
 
   nix-perl-bindings = callPackage ../src/perl/package.nix { };
 
-  nix-everything = callPackage ../packaging/everything.nix { };
+  nix-everything = callPackage ../packaging/everything.nix { } // {
+    # Note: no `passthru.overrideAllMesonComponents` etc
+    #       This would propagate into `nix.overrideAttrs f`, but then discard
+    #       `f` when `.overrideAllMesonComponents` is used.
+    #       Both "methods" should be views on the same fixpoint overriding mechanism
+    #       for that to work. For now, we intentionally don't support the broken
+    #       two-fixpoint solution.
+    /**
+      Apply an extension function (i.e. overlay-shaped) to all component derivations, and return the nix package.
+
+      Single argument: the extension function to apply (finalAttrs: prevAttrs: { ... })
+    */
+    overrideAllMesonComponents = f: (scope.overrideAllMesonComponents f).nix-everything;
+
+    /**
+      Append patches to be applied to the whole Nix source.
+      This affects all components.
+
+      Changes to the packaging expressions will be ignored.
+
+      Single argument: list of patches to apply
+
+      See also `overrideSource`
+    */
+    appendPatches = ps: (scope.appendPatches ps).nix-everything;
+
+    /**
+      Provide an alternate source. This allows the expressions to be vendored without copying the sources,
+      but it does make the build non-granular; all components will use a complete source.
+
+      Packaging expressions will be ignored.
+
+      Filesets in the packaging expressions will be ignored.
+
+      Single argument: the source to use.
+
+      See also `appendPatches`
+    */
+    overrideSource = src: (scope.overrideSource src).nix-everything;
+
+    /**
+      Override any internals of the Nix package set.
+
+      Single argument: the extension function to apply to the package set (finalScope: prevScope: { ... })
+
+      Example:
+      ```
+      overrideScope (finalScope: prevScope: { aws-sdk-cpp = null; })
+      ```
+    */
+    overrideScope = f: (scope.overrideScope f).nix-everything;
+
+  };
 }

packaging/dependencies.nix

@@ -17,11 +17,7 @@ in
 let
   inherit (pkgs) lib;
 
-  root = ../.;
-
-  stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64
-    then darwinStdenv
-    else prevStdenv;
+  stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 then darwinStdenv else prevStdenv;
 
   # Fix the following error with the default x86_64-darwin SDK:
   #
@@ -32,183 +28,76 @@ let
   # all the way back to 10.6.
   darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; };
 
-  # Nixpkgs implements this by returning a subpath into the fetched Nix sources.
-  resolvePath = p: p;
-
-  # Indirection for Nixpkgs to override when package.nix files are vendored
-  filesetToSource = lib.fileset.toSource;
-
-  /** Given a set of layers, create a mkDerivation-like function */
-  mkPackageBuilder = exts: userFn:
-    stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn);
-
-  localSourceLayer = finalAttrs: prevAttrs:
-    let
-      workDirPath =
-        # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has
-        # the requirement that everything except passthru and meta must be
-        # serialized by mkDerivation, which doesn't work for this.
-        prevAttrs.workDir;
-
-      workDirSubpath = lib.path.removePrefix root workDirPath;
-      sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset;
-      src = lib.fileset.toSource { fileset = sources; inherit root; };
-
-    in
-    {
-      sourceRoot = "${src.name}/" + workDirSubpath;
-      inherit src;
-
-      # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir.
-      fileset = null;
-      workDir = null;
-    };
-
-  mesonLayer = finalAttrs: prevAttrs:
-    {
-      # NOTE:
-      # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26,
-      # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default.
-      # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype.
-      mesonBuildType = "release";
-      # NOTE:
-      # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the
-      # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10.
-      # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable.
-      preConfigure = prevAttrs.preConfigure or "" + lib.optionalString (
-        !stdenv.hostPlatform.isWindows
-        # build failure
-        && !stdenv.hostPlatform.isStatic
-        # LTO breaks exception handling on x86-64-darwin.
-        && stdenv.system != "x86_64-darwin"
-      ) ''
-        case "$mesonBuildType" in
-        release|minsize) appendToVar mesonFlags "-Db_lto=true"  ;;
-        *)               appendToVar mesonFlags "-Db_lto=false" ;;
-        esac
-      '';
-      nativeBuildInputs = [
-        pkgs.buildPackages.meson
-        pkgs.buildPackages.ninja
-      ] ++ prevAttrs.nativeBuildInputs or [];
-      mesonCheckFlags = prevAttrs.mesonCheckFlags or [] ++ [
-        "--print-errorlogs"
-      ];
-    };
-
-  mesonBuildLayer = finalAttrs: prevAttrs:
-    {
-      nativeBuildInputs = prevAttrs.nativeBuildInputs or [] ++ [
-        pkgs.buildPackages.pkg-config
-      ];
-      separateDebugInfo = !stdenv.hostPlatform.isStatic;
-      hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
-      env = prevAttrs.env or {}
-        // lib.optionalAttrs
-          (stdenv.isLinux
-            && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")
-            && !(stdenv.hostPlatform.useLLVM or false))
-          { LDFLAGS = "-fuse-ld=gold"; };
-    };
-
-  mesonLibraryLayer = finalAttrs: prevAttrs:
-    {
-      outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ];
-    };
-
-  # Work around weird `--as-needed` linker behavior with BSD, see
-  # https://github.com/mesonbuild/meson/issues/3593
-  bsdNoLinkAsNeeded = finalAttrs: prevAttrs:
-    lib.optionalAttrs stdenv.hostPlatform.isBSD {
-      mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or [];
-    };
-
-  miscGoodPractice = finalAttrs: prevAttrs:
-    {
-      strictDeps = prevAttrs.strictDeps or true;
-      enableParallelBuilding = true;
-    };
 in
 scope: {
   inherit stdenv;
 
-  aws-sdk-cpp = (pkgs.aws-sdk-cpp.override {
-    apis = [ "s3" "transfer" ];
-    customMemoryManagement = false;
-  }).overrideAttrs {
-    # only a stripped down version is built, which takes a lot less resources
-    # to build, so we don't need a "big-parallel" machine.
-    requiredSystemFeatures = [ ];
-  };
+  aws-sdk-cpp =
+    (pkgs.aws-sdk-cpp.override {
+      apis = [
+        "s3"
+        "transfer"
+      ];
+      customMemoryManagement = false;
+    }).overrideAttrs
+      {
+        # only a stripped down version is built, which takes a lot less resources
+        # to build, so we don't need a "big-parallel" machine.
+        requiredSystemFeatures = [ ];
+      };
 
   boehmgc = pkgs.boehmgc.override {
     enableLargeConfig = true;
   };
 
   # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed.
-  boost = (pkgs.boost.override {
-    extraB2Args = [
-      "--with-container"
-      "--with-context"
-      "--with-coroutine"
-    ];
-  }).overrideAttrs (old: {
-    # Need to remove `--with-*` to use `--with-libraries=...`
-    buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase;
-    installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase;
-  });
-
-  libgit2 = pkgs.libgit2.overrideAttrs (attrs: {
-    cmakeFlags = attrs.cmakeFlags or []
-      ++ [ "-DUSE_SSH=exec" ];
-    nativeBuildInputs = attrs.nativeBuildInputs or []
-      # gitMinimal does not build on Windows. See packbuilder patch.
-      ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
-        # Needed for `git apply`; see `prePatch`
-        pkgs.buildPackages.gitMinimal
+  boost =
+    (pkgs.boost.override {
+      extraB2Args = [
+        "--with-container"
+        "--with-context"
+        "--with-coroutine"
       ];
-    # Only `git apply` can handle git binary patches
-    prePatch = attrs.prePatch or ""
-      + lib.optionalString (!stdenv.hostPlatform.isWindows) ''
-        patch() {
-          git apply
-        }
-      '';
-    patches = attrs.patches or []
-      ++ [
-        ./patches/libgit2-mempack-thin-packfile.patch
-      ]
-      # gitMinimal does not build on Windows, but fortunately this patch only
-      # impacts interruptibility
-      ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
-        # binary patch; see `prePatch`
-        ./patches/libgit2-packbuilder-callback-interruptible.patch
-      ];
-  });
+    }).overrideAttrs
+      (old: {
+        # Need to remove `--with-*` to use `--with-libraries=...`
+        buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase;
+        installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase;
+      });
 
-  inherit resolvePath filesetToSource;
-
-  mkMesonDerivation =
-    mkPackageBuilder [
-      miscGoodPractice
-      localSourceLayer
-      mesonLayer
-    ];
-  mkMesonExecutable =
-    mkPackageBuilder [
-      miscGoodPractice
-      bsdNoLinkAsNeeded
-      localSourceLayer
-      mesonLayer
-      mesonBuildLayer
-    ];
-  mkMesonLibrary =
-    mkPackageBuilder [
-      miscGoodPractice
-      bsdNoLinkAsNeeded
-      localSourceLayer
-      mesonLayer
-      mesonBuildLayer
-      mesonLibraryLayer
-    ];
+  libgit2 = pkgs.libgit2.overrideAttrs (
+    attrs:
+    {
+      cmakeFlags = attrs.cmakeFlags or [ ] ++ [ "-DUSE_SSH=exec" ];
+    }
+    # libgit2: Nixpkgs 24.11 has < 1.9.0, which needs our patches
+    // lib.optionalAttrs (!lib.versionAtLeast pkgs.libgit2.version "1.9.0") {
+      nativeBuildInputs =
+        attrs.nativeBuildInputs or [ ]
+        # gitMinimal does not build on Windows. See packbuilder patch.
+        ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
+          # Needed for `git apply`; see `prePatch`
+          pkgs.buildPackages.gitMinimal
+        ];
+      # Only `git apply` can handle git binary patches
+      prePatch =
+        attrs.prePatch or ""
+        + lib.optionalString (!stdenv.hostPlatform.isWindows) ''
+          patch() {
+            git apply
+          }
+        '';
+      patches =
+        attrs.patches or [ ]
+        ++ [
+          ./patches/libgit2-mempack-thin-packfile.patch
+        ]
+        # gitMinimal does not build on Windows, but fortunately this patch only
+        # impacts interruptibility
+        ++ lib.optionals (!stdenv.hostPlatform.isWindows) [
+          # binary patch; see `prePatch`
+          ./patches/libgit2-packbuilder-callback-interruptible.patch
+        ];
+    }
+  );
 }

packaging/dev-shell.nix

@@ -2,127 +2,131 @@
 
 { pkgs }:
 
-pkgs.nixComponents.nix-util.overrideAttrs (attrs:
+pkgs.nixComponents.nix-util.overrideAttrs (
+  attrs:
 
-let
-  stdenv = pkgs.nixDependencies.stdenv;
-  buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
-  modular = devFlake.getSystem stdenv.buildPlatform.system;
-  transformFlag = prefix: flag:
-    assert builtins.isString flag;
-    let
-      rest = builtins.substring 2 (builtins.stringLength flag) flag;
-    in
+  let
+    stdenv = pkgs.nixDependencies.stdenv;
+    buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+    modular = devFlake.getSystem stdenv.buildPlatform.system;
+    transformFlag =
+      prefix: flag:
+      assert builtins.isString flag;
+      let
+        rest = builtins.substring 2 (builtins.stringLength flag) flag;
+      in
       "-D${prefix}:${rest}";
-  havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
-  ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
-in {
-  pname = "shell-for-" + attrs.pname;
+    havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
+    ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
+  in
+  {
+    pname = "shell-for-" + attrs.pname;
 
-  # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
-  version = lib.fileContents ../.version;
-  name = attrs.pname;
+    # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
+    version = lib.fileContents ../.version;
+    name = attrs.pname;
 
-  installFlags = "sysconfdir=$(out)/etc";
-  shellHook = ''
-    PATH=$prefix/bin:$PATH
-    unset PYTHONPATH
-    export MANPATH=$out/share/man:$MANPATH
+    installFlags = "sysconfdir=$(out)/etc";
+    shellHook = ''
+      PATH=$prefix/bin:$PATH
+      unset PYTHONPATH
+      export MANPATH=$out/share/man:$MANPATH
 
-    # Make bash completion work.
-    XDG_DATA_DIRS+=:$out/share
+      # Make bash completion work.
+      XDG_DATA_DIRS+=:$out/share
 
-    # Make the default phases do the right thing.
-    # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase.
-    # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C <dir>' to ninja.
+      # Make the default phases do the right thing.
+      # FIXME: this wouldn't be needed if the ninja package set buildPhase() instead of $buildPhase.
+      # FIXME: mesonConfigurePhase shouldn't cd to the build directory. It would be better to pass '-C <dir>' to ninja.
 
-    cdToBuildDir() {
-        if [[ ! -e build.ninja ]]; then
-            cd build
-        fi
-    }
+      cdToBuildDir() {
+          if [[ ! -e build.ninja ]]; then
+              cd build
+          fi
+      }
 
-    configurePhase() {
-        mesonConfigurePhase
-    }
+      configurePhase() {
+          mesonConfigurePhase
+      }
 
-    buildPhase() {
-        cdToBuildDir
-        ninjaBuildPhase
-    }
+      buildPhase() {
+          cdToBuildDir
+          ninjaBuildPhase
+      }
 
-    checkPhase() {
-        cdToBuildDir
-        mesonCheckPhase
-    }
+      checkPhase() {
+          cdToBuildDir
+          mesonCheckPhase
+      }
 
-    installPhase() {
-        cdToBuildDir
-        ninjaInstallPhase
-    }
-  '';
+      installPhase() {
+          cdToBuildDir
+          ninjaInstallPhase
+      }
+    '';
 
-  # We use this shell with the local checkout, not unpackPhase.
-  src = null;
+    # We use this shell with the local checkout, not unpackPhase.
+    src = null;
 
-  env = {
-    # Needed for Meson to find Boost.
-    # https://github.com/NixOS/nixpkgs/issues/86131.
-    BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include";
-    BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib";
-    # For `make format`, to work without installing pre-commit
-    _NIX_PRE_COMMIT_HOOKS_CONFIG =
-      "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}";
-  };
+    env = {
+      # For `make format`, to work without installing pre-commit
+      _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml"
+        modular.pre-commit.settings.rawConfig
+      }";
+    };
 
-  mesonFlags =
-    map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags)
-    ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags)
-    ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags)
-    ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags))
-    ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags)
-    ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags)
-    ;
+    mesonFlags =
+      map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags)
+      ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags)
+      ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags)
+      ++ lib.optionals havePerl (
+        map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)
+      )
+      ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags)
+      ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags);
 
-  nativeBuildInputs = attrs.nativeBuildInputs or []
-    ++ pkgs.nixComponents.nix-util.nativeBuildInputs
-    ++ pkgs.nixComponents.nix-store.nativeBuildInputs
-    ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
-    ++ pkgs.nixComponents.nix-expr.nativeBuildInputs
-    ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
-    ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs
-    ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
-    ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
-    ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs
-    ++ lib.optional
-      (!buildCanExecuteHost
-         # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
-         && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin)
-         && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages
-         && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages))
-      pkgs.buildPackages.mesonEmulatorHook
-    ++ [
-      pkgs.buildPackages.cmake
-      pkgs.buildPackages.shellcheck
-      pkgs.buildPackages.changelog-d
-      modular.pre-commit.settings.package
-      (pkgs.writeScriptBin "pre-commit-hooks-install"
-        modular.pre-commit.settings.installationScript)
-    ]
-    # TODO: Remove the darwin check once
-    # https://github.com/NixOS/nixpkgs/pull/291814 is available
-    ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
-    ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (lib.hiPrio pkgs.buildPackages.clang-tools);
+    nativeBuildInputs =
+      attrs.nativeBuildInputs or [ ]
+      ++ pkgs.nixComponents.nix-util.nativeBuildInputs
+      ++ pkgs.nixComponents.nix-store.nativeBuildInputs
+      ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs
+      ++ pkgs.nixComponents.nix-expr.nativeBuildInputs
+      ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs
+      ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs
+      ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs
+      ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs
+      ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs
+      ++ lib.optional (
+        !buildCanExecuteHost
+        # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
+        && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin)
+        && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages
+        && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)
+      ) pkgs.buildPackages.mesonEmulatorHook
+      ++ [
+        pkgs.buildPackages.cmake
+        pkgs.buildPackages.shellcheck
+        pkgs.buildPackages.changelog-d
+        modular.pre-commit.settings.package
+        (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
+      ]
+      # TODO: Remove the darwin check once
+      # https://github.com/NixOS/nixpkgs/pull/291814 is available
+      ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear
+      ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (
+        lib.hiPrio pkgs.buildPackages.clang-tools
+      );
 
-  buildInputs = attrs.buildInputs or []
-    ++ pkgs.nixComponents.nix-util.buildInputs
-    ++ pkgs.nixComponents.nix-store.buildInputs
-    ++ pkgs.nixComponents.nix-store-tests.externalBuildInputs
-    ++ pkgs.nixComponents.nix-fetchers.buildInputs
-    ++ pkgs.nixComponents.nix-expr.buildInputs
-    ++ pkgs.nixComponents.nix-expr.externalPropagatedBuildInputs
-    ++ pkgs.nixComponents.nix-cmd.buildInputs
-    ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.externalBuildInputs
-    ++ lib.optional havePerl pkgs.perl
-    ;
-})
+    buildInputs =
+      attrs.buildInputs or [ ]
+      ++ pkgs.nixComponents.nix-util.buildInputs
+      ++ pkgs.nixComponents.nix-store.buildInputs
+      ++ pkgs.nixComponents.nix-store-tests.externalBuildInputs
+      ++ pkgs.nixComponents.nix-fetchers.buildInputs
+      ++ pkgs.nixComponents.nix-expr.buildInputs
+      ++ pkgs.nixComponents.nix-expr.externalPropagatedBuildInputs
+      ++ pkgs.nixComponents.nix-cmd.buildInputs
+      ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.externalBuildInputs
+      ++ lib.optional havePerl pkgs.perl;
+  }
+)

packaging/everything.nix

@@ -1,8 +1,11 @@
 {
   lib,
   stdenv,
+  lndir,
   buildEnv,
 
+  maintainers,
+
   nix-util,
   nix-util-c,
   nix-util-tests,
@@ -38,74 +41,37 @@
   nix-perl-bindings,
 
   testers,
-  runCommand,
+
+  patchedSrc ? null,
 }:
 
 let
-  libs = {
-    inherit
-      nix-util
-      nix-util-c
-      nix-store
-      nix-store-c
-      nix-fetchers
-      nix-expr
-      nix-expr-c
-      nix-flake
-      nix-flake-c
-      nix-main
-      nix-main-c
-      nix-cmd
-    ;
-  } // lib.optionalAttrs (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) {
-    # Currently fails in static build
-    inherit
-      nix-perl-bindings
-    ;
-  };
-
-  dev = stdenv.mkDerivation (finalAttrs: {
-    name = "nix-${nix-cli.version}-dev";
-    pname = "nix";
-    version = nix-cli.version;
-    dontUnpack = true;
-    dontBuild = true;
-    libs = map lib.getDev (lib.attrValues libs);
-    installPhase = ''
-      mkdir -p $out/nix-support
-      echo $libs >> $out/nix-support/propagated-build-inputs
-    '';
-    passthru = {
-      tests = {
-        pkg-config =
-          testers.hasPkgConfigModules {
-            package = finalAttrs.finalPackage;
-          };
+  libs =
+    {
+      inherit
+        nix-util
+        nix-util-c
+        nix-store
+        nix-store-c
+        nix-fetchers
+        nix-expr
+        nix-expr-c
+        nix-flake
+        nix-flake-c
+        nix-main
+        nix-main-c
+        nix-cmd
+        ;
+    }
+    // lib.optionalAttrs
+      (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform)
+      {
+        # Currently fails in static build
+        inherit
+          nix-perl-bindings
+          ;
       };
 
-      # If we were to fully emulate output selection here, we'd confuse the Nix CLIs,
-      # because they rely on `drvPath`.
-      dev = finalAttrs.finalPackage.out;
-
-      libs = throw "`nix.dev.libs` is not meant to be used; use `nix.libs` instead.";
-    };
-    meta = {
-      pkgConfigModules = [
-        "nix-cmd"
-        "nix-expr"
-        "nix-expr-c"
-        "nix-fetchers"
-        "nix-flake"
-        "nix-flake-c"
-        "nix-main"
-        "nix-main-c"
-        "nix-store"
-        "nix-store-c"
-        "nix-util"
-        "nix-util-c"
-      ];
-    };
-  });
   devdoc = buildEnv {
     name = "nix-${nix-cli.version}-devdoc";
     paths = [
@@ -115,39 +81,100 @@ let
   };
 
 in
-(buildEnv {
-  name = "nix-${nix-cli.version}";
-  paths = [
-    nix-cli
-    nix-manual.man
+stdenv.mkDerivation (finalAttrs: {
+  pname = "nix";
+  version = nix-cli.version;
+
+  /**
+    This package uses a multi-output derivation, even though some outputs could
+    have been provided directly by the constituent component that provides it.
+
+    This is because not all tooling handles packages composed of arbitrary
+    outputs yet. This includes nix itself, https://github.com/NixOS/nix/issues/6507.
+
+    `devdoc` is also available, but not listed here, because this attribute is
+    not an output of the same derivation that provides `out`, `dev`, etc.
+  */
+  outputs = [
+    "out"
+    "dev"
+    "doc"
+    "man"
   ];
 
-  meta.mainProgram = "nix";
-}).overrideAttrs (finalAttrs: prevAttrs: {
+  /**
+    Unpacking is handled in this package's constituent components
+  */
+  dontUnpack = true;
+  /**
+    Building is handled in this package's constituent components
+  */
+  dontBuild = true;
+
+  /**
+    `doCheck` controles whether tests are added as build gate for the combined package.
+    This includes both the unit tests and the functional tests, but not the
+    integration tests that run in CI (the flake's `hydraJobs` and some of the `checks`).
+  */
   doCheck = true;
-  doInstallCheck = true;
 
-  checkInputs = [
-    # Make sure the unit tests have passed
-    nix-util-tests.tests.run
-    nix-store-tests.tests.run
-    nix-expr-tests.tests.run
-    nix-fetchers-tests.tests.run
-    nix-flake-tests.tests.run
+  /**
+    `fixupPhase` currently doesn't understand that a symlink output isn't writable.
 
-    # Make sure the functional tests have passed
-    nix-functional-tests
+    We don't compile or link anything in this derivation, so fixups aren't needed.
+  */
+  dontFixup = true;
 
-    # dev bundle is ok
-    # (checkInputs must be empty paths??)
-    (runCommand "check-pkg-config" { checked = dev.tests.pkg-config; } "mkdir $out")
-  ] ++ lib.optionals (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform) [
-    # Perl currently fails in static build
-    # TODO: Split out tests into a separate derivation?
-    nix-perl-bindings
+  checkInputs =
+    [
+      # Make sure the unit tests have passed
+      nix-util-tests.tests.run
+      nix-store-tests.tests.run
+      nix-expr-tests.tests.run
+      nix-fetchers-tests.tests.run
+      nix-flake-tests.tests.run
+
+      # Make sure the functional tests have passed
+      nix-functional-tests
+    ]
+    ++ lib.optionals
+      (!stdenv.hostPlatform.isStatic && stdenv.buildPlatform.canExecute stdenv.hostPlatform)
+      [
+        # Perl currently fails in static build
+        # TODO: Split out tests into a separate derivation?
+        nix-perl-bindings
+      ];
+
+  nativeBuildInputs = [
+    lndir
   ];
-  passthru = prevAttrs.passthru // {
+
+  installPhase =
+    let
+      devPaths = lib.mapAttrsToList (_k: lib.getDev) finalAttrs.finalPackage.libs;
+    in
+    ''
+      mkdir -p $out $dev/nix-support
+
+      # Custom files
+      echo $libs >> $dev/nix-support/propagated-build-inputs
+      echo ${nix-cli} ${lib.escapeShellArgs devPaths} >> $dev/nix-support/propagated-build-inputs
+
+      # Merged outputs
+      lndir ${nix-cli} $out
+
+      for lib in ${lib.escapeShellArgs devPaths}; do
+        lndir $lib $dev
+      done
+
+      # Forwarded outputs
+      ln -sT ${nix-manual} $doc
+      ln -sT ${nix-manual.man} $man
+    '';
+
+  passthru = {
     inherit (nix-cli) version;
+    src = patchedSrc;
 
     /**
       These are the libraries that are part of the Nix project. They are used
@@ -165,28 +192,53 @@ in
         unusedInputsForTests = [ nix ];
         disallowedReferences = nix.all;
       ```
-     */
+    */
     inherit libs;
 
-    tests = prevAttrs.passthru.tests or {} // {
-      # TODO: create a proper fixpoint and:
-      # pkg-config =
-      #   testers.hasPkgConfigModules {
-      #     package = finalPackage;
-      #   };
-    };
+    /**
+      Developer documentation for `nix`, in `share/doc/nix/{internal,external}-api/`.
+
+      This is not a proper output; see `outputs` for context.
+    */
+    inherit devdoc;
 
     /**
-      A derivation referencing the `dev` outputs of the Nix libraries.
-     */
-    inherit dev;
-    inherit devdoc;
-    doc = nix-manual;
-    outputs = [ "out" "dev" "devdoc" "doc" ];
-    all = lib.attrValues (lib.genAttrs finalAttrs.passthru.outputs (outName: finalAttrs.finalPackage.${outName}));
+      Extra tests that test this package, but do not run as part of the build.
+      See <https://nixos.org/manual/nixpkgs/stable/index.html#var-passthru-tests>
+    */
+    tests = {
+      pkg-config = testers.hasPkgConfigModules {
+        package = finalAttrs.finalPackage;
+      };
+    };
   };
-  meta = prevAttrs.meta // {
+
+  meta = {
+    mainProgram = "nix";
     description = "The Nix package manager";
-    pkgConfigModules = dev.meta.pkgConfigModules;
+    longDescription = nix-cli.meta.longDescription;
+    homepage = nix-cli.meta.homepage;
+    license = nix-cli.meta.license;
+    maintainers = maintainers;
+    platforms = nix-cli.meta.platforms;
+    outputsToInstall = [
+      "out"
+      "man"
+    ];
+    pkgConfigModules = [
+      "nix-cmd"
+      "nix-expr"
+      "nix-expr-c"
+      "nix-fetchers"
+      "nix-flake"
+      "nix-flake-c"
+      "nix-main"
+      "nix-main-c"
+      "nix-store"
+      "nix-store-c"
+      "nix-util"
+      "nix-util-c"
+    ];
   };
+
 })

packaging/hydra.nix

@@ -1,22 +1,25 @@
-{ inputs
-, binaryTarball
-, forAllCrossSystems
-, forAllSystems
-, lib
-, linux64BitSystems
-, nixpkgsFor
-, self
-, officialRelease
+{
+  inputs,
+  binaryTarball,
+  forAllCrossSystems,
+  forAllSystems,
+  lib,
+  linux64BitSystems,
+  nixpkgsFor,
+  self,
+  officialRelease,
 }:
 let
   inherit (inputs) nixpkgs nixpkgs-regression;
 
-  installScriptFor = tarballs:
+  installScriptFor =
+    tarballs:
     nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix {
       inherit tarballs;
     };
 
-  testNixVersions = pkgs: daemon:
+  testNixVersions =
+    pkgs: daemon:
     pkgs.nixComponents.nix-functional-tests.override {
       pname = "nix-daemon-compat-tests";
       version = "${pkgs.nix.version}-with-daemon-${daemon.version}";
@@ -54,44 +57,70 @@ let
 in
 {
   # Binary package for various platforms.
-  build = forAllPackages (pkgName:
-    forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName}));
+  build = forAllPackages (
+    pkgName: forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})
+  );
 
-  shellInputs = removeAttrs
-    (forAllSystems (system: self.devShells.${system}.default.inputDerivation))
-    [ "i686-linux" ];
+  shellInputs = removeAttrs (forAllSystems (
+    system: self.devShells.${system}.default.inputDerivation
+  )) [ "i686-linux" ];
 
-  buildStatic = forAllPackages (pkgName:
-    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName}));
+  buildStatic = forAllPackages (
+    pkgName:
+    lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})
+  );
 
-  buildCross = forAllPackages (pkgName:
+  buildCross = forAllPackages (
+    pkgName:
     # Hack to avoid non-evaling package
-    (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id)
-    (forAllCrossSystems (crossSystem:
-      lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))));
+    (
+      if pkgName == "nix-functional-tests" then
+        lib.flip builtins.removeAttrs [ "x86_64-w64-mingw32" ]
+      else
+        lib.id
+    )
+      (
+        forAllCrossSystems (
+          crossSystem:
+          lib.genAttrs [ "x86_64-linux" ] (
+            system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}
+          )
+        )
+      )
+  );
 
-  buildNoGc = let
-    components = forAllSystems (system:
-      nixpkgsFor.${system}.native.nixComponents.overrideScope (self: super: {
-        nix-expr = super.nix-expr.override { enableGC = false; };
-      })
-    );
-  in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
+  buildNoGc =
+    let
+      components = forAllSystems (
+        system:
+        nixpkgsFor.${system}.native.nixComponents.overrideScope (
+          self: super: {
+            nix-expr = super.nix-expr.override { enableGC = false; };
+          }
+        )
+      );
+    in
+    forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
 
   buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli);
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.
-  buildReadlineNoMarkdown = let
-    components = forAllSystems (system:
-      nixpkgsFor.${system}.native.nixComponents.overrideScope (self: super: {
-        nix-cmd = super.nix-cmd.override {
-          enableMarkdown = false;
-          readlineFlavor = "readline";
-        };
-      })
-    );
-  in forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
+  buildReadlineNoMarkdown =
+    let
+      components = forAllSystems (
+        system:
+        nixpkgsFor.${system}.native.nixComponents.overrideScope (
+          self: super: {
+            nix-cmd = super.nix-cmd.override {
+              enableMarkdown = false;
+              readlineFlavor = "readline";
+            };
+          }
+        )
+      );
+    in
+    forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
 
   # Perl bindings for various platforms.
   perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings);
@@ -99,13 +128,18 @@ in
   # Binary tarball for various platforms, containing a Nix store
   # with the closure of 'nix' package, and the second half of
   # the installation script.
-  binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native);
+  binaryTarball = forAllSystems (
+    system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native
+  );
 
-  binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system:
-    forAllCrossSystems (crossSystem:
-      binaryTarball
-        nixpkgsFor.${system}.cross.${crossSystem}.nix
-        nixpkgsFor.${system}.cross.${crossSystem}));
+  binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (
+    system:
+    forAllCrossSystems (
+      crossSystem:
+      binaryTarball nixpkgsFor.${system}.cross.${crossSystem}.nix
+        nixpkgsFor.${system}.cross.${crossSystem}
+    )
+  );
 
   # The first half of the installation script. This is uploaded
   # to https://nixos.org/nix/install. It downloads the binary
@@ -124,9 +158,12 @@ in
     self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu"
   ];
 
-  installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix {
-    tarballs = [ self.hydraJobs.binaryTarball.${system} ];
-  });
+  installerScriptForGHA = forAllSystems (
+    system:
+    nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix {
+      tarballs = [ self.hydraJobs.binaryTarball.${system} ];
+    }
+  );
 
   # docker image with Nix inside
   dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage);
@@ -147,16 +184,24 @@ in
   external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs;
 
   # System tests.
-  tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // {
+  tests =
+    import ../tests/nixos {
+      inherit
+        lib
+        nixpkgs
+        nixpkgsFor
+        self
+        ;
+    }
+    // {
 
-    # Make sure that nix-env still produces the exact same result
-    # on a particular version of Nixpkgs.
-    evalNixpkgs =
-      let
-        inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
-      in
-      runCommand "eval-nixos" { buildInputs = [ nix ]; }
-        ''
+      # Make sure that nix-env still produces the exact same result
+      # on a particular version of Nixpkgs.
+      evalNixpkgs =
+        let
+          inherit (nixpkgsFor.x86_64-linux.native) runCommand nix;
+        in
+        runCommand "eval-nixos" { buildInputs = [ nix ]; } ''
           type -p nix-env
           # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593.
           (
@@ -167,36 +212,36 @@ in
           mkdir $out
         '';
 
-    nixpkgsLibTests =
-      forAllSystems (system:
-        import (nixpkgs + "/lib/tests/test-with-nix.nix")
-          {
-            lib = nixpkgsFor.${system}.native.lib;
-            nix = self.packages.${system}.nix-cli;
-            pkgs = nixpkgsFor.${system}.native;
-          }
+      nixpkgsLibTests = forAllSystems (
+        system:
+        import (nixpkgs + "/lib/tests/test-with-nix.nix") {
+          lib = nixpkgsFor.${system}.native.lib;
+          nix = self.packages.${system}.nix-cli;
+          pkgs = nixpkgsFor.${system}.native;
+        }
       );
-  };
+    };
 
   metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" {
     pkgs = nixpkgsFor.x86_64-linux.native;
     nixpkgs = nixpkgs-regression;
   };
 
-  installTests = forAllSystems (system:
-    let pkgs = nixpkgsFor.${system}.native; in
-    pkgs.runCommand "install-tests"
-      {
-        againstSelf = testNixVersions pkgs pkgs.nix;
-        againstCurrentLatest =
-          # FIXME: temporarily disable this on macOS because of #3605.
-          if system == "x86_64-linux"
-          then testNixVersions pkgs pkgs.nixVersions.latest
-          else null;
-        # Disabled because the latest stable version doesn't handle
-        # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
-        # againstLatestStable = testNixVersions pkgs pkgs.nixStable;
-      } "touch $out");
+  installTests = forAllSystems (
+    system:
+    let
+      pkgs = nixpkgsFor.${system}.native;
+    in
+    pkgs.runCommand "install-tests" {
+      againstSelf = testNixVersions pkgs pkgs.nix;
+      againstCurrentLatest =
+        # FIXME: temporarily disable this on macOS because of #3605.
+        if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null;
+      # Disabled because the latest stable version doesn't handle
+      # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work
+      # againstLatestStable = testNixVersions pkgs pkgs.nixStable;
+    } "touch $out"
+  );
 
   installerTests = import ../tests/installer {
     binaryTarballs = self.hydraJobs.binaryTarball;

scripts/binary-tarball.nix

@@ -1,14 +1,18 @@
-{ runCommand
-, system
-, buildPackages
-, cacert
-, nix
+{
+  runCommand,
+  system,
+  buildPackages,
+  cacert,
+  nix,
 }:
 
 let
 
   installerClosureInfo = buildPackages.closureInfo {
-    rootPaths = [ nix cacert ];
+    rootPaths = [
+      nix
+      cacert
+    ];
   };
 
   inherit (nix) version;

scripts/installer.nix

@@ -1,36 +1,42 @@
-{ lib
-, runCommand
-, nix
-, tarballs
+{
+  lib,
+  runCommand,
+  nix,
+  tarballs,
 }:
 
-runCommand "installer-script" {
-  buildInputs = [ nix ];
-} ''
-  mkdir -p $out/nix-support
-
-  # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
-  tarballPath() {
-    # Remove the store prefix
-    local path=''${1#${builtins.storeDir}/}
-    # Get the path relative to the derivation root
-    local rest=''${path#*/}
-    # Get the derivation hash
-    local drvHash=''${path%%-*}
-    echo "$drvHash/$rest"
+runCommand "installer-script"
+  {
+    buildInputs = [ nix ];
   }
+  ''
+    mkdir -p $out/nix-support
 
-  substitute ${./install.in} $out/install \
-    ${lib.concatMapStrings
-      (tarball: let
-          inherit (tarball.stdenv.hostPlatform) system;
-        in '' \
-        --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
-        --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
-        ''
-      )
-      tarballs
-    } --replace '@nixVersion@' ${nix.version}
+    # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix.
+    tarballPath() {
+      # Remove the store prefix
+      local path=''${1#${builtins.storeDir}/}
+      # Get the path relative to the derivation root
+      local rest=''${path#*/}
+      # Get the derivation hash
+      local drvHash=''${path%%-*}
+      echo "$drvHash/$rest"
+    }
 
-  echo "file installer $out/install" >> $out/nix-support/hydra-build-products
-''
+    substitute ${./install.in} $out/install \
+      ${
+        lib.concatMapStrings (
+          tarball:
+          let
+            inherit (tarball.stdenv.hostPlatform) system;
+          in
+          ''
+            \
+                   --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \
+                   --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \
+          ''
+        ) tarballs
+      } --replace '@nixVersion@' ${nix.version}
+
+    echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+  ''

scripts/nix-profile-daemon.sh.in

@@ -1,7 +1,7 @@
 # Only execute this file once per shell.
 # This file is tested by tests/installer/default.nix.
 if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
-__ETC_PROFILE_NIX_SOURCED=1
+export __ETC_PROFILE_NIX_SOURCED=1
 
 NIX_LINK=$HOME/.nix-profile
 if [ -n "${XDG_STATE_HOME-}" ]; then

scripts/nix-profile.fish.in

@@ -29,7 +29,7 @@ if test -n "$HOME" && test -n "$USER"
     end
 
     # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
-    if test -n "$NIX_SSH_CERT_FILE"
+    if test -n "$NIX_SSL_CERT_FILE"
         : # Allow users to override the NIX_SSL_CERT_FILE
     else if test -e /etc/ssl/certs/ca-certificates.crt # NixOS, Ubuntu, Debian, Gentoo, Arch
         set --export NIX_SSL_CERT_FILE /etc/ssl/certs/ca-certificates.crt

src/external-api-docs/package.nix

@@ -1,11 +1,12 @@
-{ lib
-, mkMesonDerivation
+{
+  lib,
+  mkMesonDerivation,
 
-, doxygen
+  doxygen,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -39,11 +40,10 @@ mkMesonDerivation (finalAttrs: {
     doxygen
   ];
 
-  preConfigure =
-    ''
-      chmod u+w ./.version
-      echo ${finalAttrs.version} > ./.version
-    '';
+  preConfigure = ''
+    chmod u+w ./.version
+    echo ${finalAttrs.version} > ./.version
+  '';
 
   postInstall = ''
     mkdir -p ''${!outputDoc}/nix-support

src/internal-api-docs/package.nix

@@ -1,11 +1,12 @@
-{ lib
-, mkMesonDerivation
+{
+  lib,
+  mkMesonDerivation,
 
-, doxygen
+  doxygen,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -17,27 +18,28 @@ mkMesonDerivation (finalAttrs: {
   inherit version;
 
   workDir = ./.;
-  fileset = let
-    cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh");
-  in fileset.unions [
-    ./.version
-    ../../.version
-    ./meson.build
-    ./doxygen.cfg.in
-    # Source is not compiled, but still must be available for Doxygen
-    # to gather comments.
-    (cpp ../.)
-  ];
+  fileset =
+    let
+      cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh");
+    in
+    fileset.unions [
+      ./.version
+      ../../.version
+      ./meson.build
+      ./doxygen.cfg.in
+      # Source is not compiled, but still must be available for Doxygen
+      # to gather comments.
+      (cpp ../.)
+    ];
 
   nativeBuildInputs = [
     doxygen
   ];
 
-  preConfigure =
-    ''
-      chmod u+w ./.version
-      echo ${finalAttrs.version} > ./.version
-    '';
+  preConfigure = ''
+    chmod u+w ./.version
+    echo ${finalAttrs.version} > ./.version
+  '';
 
   postInstall = ''
     mkdir -p ''${!outputDoc}/nix-support

src/libcmd/meson.build

@@ -113,10 +113,15 @@ headers = [config_h] + files(
   'repl.hh',
 )
 
+subdir('nix-meson-build-support/export-all-symbols')
+subdir('nix-meson-build-support/windows-version')
+
 this_library = library(
   'nixcmd',
   sources,
   dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
   prelink : true, # For C++ static initializers
   install : true,
 )

src/libcmd/package.nix

@@ -1,32 +1,33 @@
-{ lib
-, stdenv
-, mkMesonLibrary
+{
+  lib,
+  stdenv,
+  mkMesonLibrary,
 
-, nix-util
-, nix-store
-, nix-fetchers
-, nix-expr
-, nix-flake
-, nix-main
-, editline
-, readline
-, lowdown
-, nlohmann_json
+  nix-util,
+  nix-store,
+  nix-fetchers,
+  nix-expr,
+  nix-flake,
+  nix-main,
+  editline,
+  readline,
+  lowdown,
+  nlohmann_json,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 
-# Whether to enable Markdown rendering in the Nix binary.
-, enableMarkdown ? !stdenv.hostPlatform.isWindows
+  # Whether to enable Markdown rendering in the Nix binary.
+  enableMarkdown ? !stdenv.hostPlatform.isWindows,
 
-# Which interactive line editor library to use for Nix's repl.
-#
-# Currently supported choices are:
-#
-# - editline (default)
-# - readline
-, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
+  # Which interactive line editor library to use for Nix's repl.
+  #
+  # Currently supported choices are:
+  #
+  # - editline (default)
+  # - readline
+  readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline",
 }:
 
 let
@@ -63,14 +64,6 @@ mkMesonLibrary (finalAttrs: {
     nlohmann_json
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
     (lib.mesonEnable "markdown" enableMarkdown)
     (lib.mesonOption "readline-flavor" readlineFlavor)

src/libcmd/repl.cc

@@ -138,16 +138,13 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi
         out << ANSI_RED "error: " << ANSI_NORMAL;
     out << dt.hint.str() << "\n";
 
-    // prefer direct pos, but if noPos then try the expr.
-    auto pos = dt.pos
-        ? dt.pos
-        : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos];
+    auto pos = dt.getPos(positions);
 
     if (pos) {
-        out << *pos;
-        if (auto loc = pos->getCodeLines()) {
+        out << pos;
+        if (auto loc = pos.getCodeLines()) {
             out << "\n";
-            printCodeLines(out, "", *pos, *loc);
+            printCodeLines(out, "", pos, *loc);
             out << "\n";
         }
     }

src/libexpr-c/nix_api_value.cc

@@ -613,12 +613,8 @@ nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * st
         context->last_err_code = NIX_OK;
     try {
         auto & v = check_value_in(value);
-        nix::NixStringContext stringContext;
-        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
         nix::StorePathSet storePaths;
-        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
-
-        auto s = nix::rewriteStrings(rawStr, rewrites);
+        auto s = state->state.realiseString(v, &storePaths, isIFD);
 
         // Convert to the C API StorePath type and convert to vector for index-based access
         std::vector<StorePath> vec;

src/libexpr-c/package.nix

@@ -1,12 +1,13 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-store-c
-, nix-expr
+  nix-store-c,
+  nix-expr,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -35,14 +36,6 @@ mkMesonLibrary (finalAttrs: {
     nix-expr
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libexpr-test-support/package.nix

@@ -1,15 +1,16 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-store-test-support
-, nix-expr
-, nix-expr-c
+  nix-store-test-support,
+  nix-expr,
+  nix-expr-c,
 
-, rapidcheck
+  rapidcheck,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -39,14 +40,6 @@ mkMesonLibrary (finalAttrs: {
     rapidcheck
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libexpr-tests/package.nix

@@ -1,20 +1,21 @@
-{ lib
-, buildPackages
-, stdenv
-, mkMesonExecutable
+{
+  lib,
+  buildPackages,
+  stdenv,
+  mkMesonExecutable,
 
-, nix-expr
-, nix-expr-c
-, nix-expr-test-support
+  nix-expr,
+  nix-expr-c,
+  nix-expr-test-support,
 
-, rapidcheck
-, gtest
-, runCommand
+  rapidcheck,
+  gtest,
+  runCommand,
 
-# Configuration Options
+  # Configuration Options
 
-, version
-, resolvePath
+  version,
+  resolvePath,
 }:
 
 let
@@ -45,29 +46,27 @@ mkMesonExecutable (finalAttrs: {
     gtest
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 
   passthru = {
     tests = {
-      run = runCommand "${finalAttrs.pname}-run" {
-        meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
-      } (lib.optionalString stdenv.hostPlatform.isWindows ''
-        export HOME="$PWD/home-dir"
-        mkdir -p "$HOME"
-      '' + ''
-        export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
-        ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
-        touch $out
-      '');
+      run =
+        runCommand "${finalAttrs.pname}-run"
+          {
+            meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
+          }
+          (
+            lib.optionalString stdenv.hostPlatform.isWindows ''
+              export HOME="$PWD/home-dir"
+              mkdir -p "$HOME"
+            ''
+            + ''
+              export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
+              ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
+              touch $out
+            ''
+          );
     };
   };
 

src/libexpr/call-flake.nix

@@ -20,77 +20,77 @@ let
   # Resolve a input spec into a node name. An input spec is
   # either a node name, or a 'follows' path from the root
   # node.
-  resolveInput = inputSpec:
-    if builtins.isList inputSpec
-    then getInputByPath lockFile.root inputSpec
-    else inputSpec;
+  resolveInput =
+    inputSpec: if builtins.isList inputSpec then getInputByPath lockFile.root inputSpec else inputSpec;
 
   # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
   # root node, returning the final node.
-  getInputByPath = nodeName: path:
-    if path == []
-    then nodeName
+  getInputByPath =
+    nodeName: path:
+    if path == [ ] then
+      nodeName
     else
       getInputByPath
         # Since this could be a 'follows' input, call resolveInput.
         (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
         (builtins.tail path);
 
-  allNodes =
-    builtins.mapAttrs
-      (key: node:
-        let
+  allNodes = builtins.mapAttrs (
+    key: node:
+    let
 
-          parentNode = allNodes.${getInputByPath lockFile.root node.parent};
+      parentNode = allNodes.${getInputByPath lockFile.root node.parent};
 
-          sourceInfo =
-            if overrides ? ${key}
-            then
-              overrides.${key}.sourceInfo
-            else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/"
-            then
-              parentNode.sourceInfo // {
-                outPath = parentNode.outPath + ("/" + node.locked.path);
-              }
-            else
-              # FIXME: remove obsolete node.info.
-              # Note: lock file entries are always final.
-              fetchTreeFinal (node.info or {} // removeAttrs node.locked ["dir"]);
+      sourceInfo =
+        if overrides ? ${key} then
+          overrides.${key}.sourceInfo
+        else if node.locked.type == "path" && builtins.substring 0 1 node.locked.path != "/" then
+          parentNode.sourceInfo
+          // {
+            outPath = parentNode.outPath + ("/" + node.locked.path);
+          }
+        else
+          # FIXME: remove obsolete node.info.
+          # Note: lock file entries are always final.
+          fetchTreeFinal (node.info or { } // removeAttrs node.locked [ "dir" ]);
 
-          subdir = overrides.${key}.dir or node.locked.dir or "";
+      subdir = overrides.${key}.dir or node.locked.dir or "";
 
-          outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
+      outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir);
 
-          flake = import (outPath + "/flake.nix");
+      flake = import (outPath + "/flake.nix");
 
-          inputs = builtins.mapAttrs
-            (inputName: inputSpec: allNodes.${resolveInput inputSpec})
-            (node.inputs or {});
+      inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) (
+        node.inputs or { }
+      );
 
-          outputs = flake.outputs (inputs // { self = result; });
+      outputs = flake.outputs (inputs // { self = result; });
 
-          result =
-            outputs
-            # We add the sourceInfo attribute for its metadata, as they are
-            # relevant metadata for the flake. However, the outPath of the
-            # sourceInfo does not necessarily match the outPath of the flake,
-            # as the flake may be in a subdirectory of a source.
-            # This is shadowed in the next //
-            // sourceInfo
-            // {
-              # This shadows the sourceInfo.outPath
-              inherit outPath;
+      result =
+        outputs
+        # We add the sourceInfo attribute for its metadata, as they are
+        # relevant metadata for the flake. However, the outPath of the
+        # sourceInfo does not necessarily match the outPath of the flake,
+        # as the flake may be in a subdirectory of a source.
+        # This is shadowed in the next //
+        // sourceInfo
+        // {
+          # This shadows the sourceInfo.outPath
+          inherit outPath;
 
-              inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake";
-            };
+          inherit inputs;
+          inherit outputs;
+          inherit sourceInfo;
+          _type = "flake";
+        };
 
-        in
-          if node.flake or true then
-            assert builtins.isFunction flake.outputs;
-            result
-          else
-            sourceInfo
-      )
-      lockFile.nodes;
+    in
+    if node.flake or true then
+      assert builtins.isFunction flake.outputs;
+      result
+    else
+      sourceInfo
+  ) lockFile.nodes;
 
-in allNodes.${lockFile.root}
+in
+allNodes.${lockFile.root}

src/libexpr/eval-error.cc

@@ -45,7 +45,7 @@ EvalErrorBuilder<T> & EvalErrorBuilder<T>::withFrame(const Env & env, const Expr
     // TODO: check compatibility with nested debugger calls.
     // TODO: What side-effects??
     error.state.debugTraces.push_front(DebugTrace{
-        .pos = error.state.positions[expr.getPos()],
+        .pos = expr.getPos(),
         .expr = expr,
         .env = env,
         .hint = HintFmt("Fake frame for debugging purposes"),

src/libexpr/eval-inline.hh

@@ -146,7 +146,7 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e
 [[gnu::always_inline]]
 inline CallDepth EvalState::addCallDepth(const PosIdx pos) {
     if (callDepth > settings.maxCallDepth)
-        error<EvalError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
+        error<EvalBaseError>("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow();
 
     return CallDepth(callDepth);
 };

src/libexpr/eval-settings.cc

@@ -57,7 +57,7 @@ Strings EvalSettings::getDefaultNixPath()
 {
     Strings res;
     auto add = [&](const Path & p, const std::string & s = std::string()) {
-        if (pathAccessible(p)) {
+        if (std::filesystem::exists(p)) {
             if (s.empty()) {
                 res.push_back(p);
             } else {

src/libexpr/eval.cc

@@ -754,18 +754,26 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr &
     if (!debugRepl || inDebugger)
         return;
 
-    auto dts =
-        error && expr.getPos()
-        ? std::make_unique<DebugTraceStacker>(
-            *this,
-            DebugTrace {
-                .pos = error->info().pos ? error->info().pos : positions[expr.getPos()],
+    auto dts = [&]() -> std::unique_ptr<DebugTraceStacker> {
+        if (error && expr.getPos()) {
+            auto trace = DebugTrace{
+                .pos = [&]() -> std::variant<Pos, PosIdx> {
+                    if (error->info().pos) {
+                        if (auto * pos = error->info().pos.get())
+                            return *pos;
+                        return noPos;
+                    }
+                    return expr.getPos();
+                }(),
                 .expr = expr,
                 .env = env,
                 .hint = error->info().msg,
-                .isError = true
-            })
-        : nullptr;
+                .isError = true};
+
+            return std::make_unique<DebugTraceStacker>(*this, std::move(trace));
+        }
+        return nullptr;
+    }();
 
     if (error)
     {
@@ -810,7 +818,7 @@ static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker(
     EvalState & state,
     Expr & expr,
     Env & env,
-    std::shared_ptr<Pos> && pos,
+    std::variant<Pos, PosIdx> pos,
     const Args & ... formatArgs)
 {
     return std::make_unique<DebugTraceStacker>(state,
@@ -1087,7 +1095,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial)
                 *this,
                 *e,
                 this->baseEnv,
-                e->getPos() ? std::make_shared<Pos>(positions[e->getPos()]) : nullptr,
+                e->getPos(),
                 "while evaluating the file '%1%':", resolvedPath.to_string())
             : nullptr;
 
@@ -1313,9 +1321,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
             state,
             *this,
             env2,
-            getPos()
-                ? std::make_shared<Pos>(state.positions[getPos()])
-                : nullptr,
+            getPos(),
             "while evaluating a '%1%' expression",
             "let"
         )
@@ -1384,7 +1390,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
                 state,
                 *this,
                 env,
-                state.positions[getPos()],
+                getPos(),
                 "while evaluating the attribute '%1%'",
                 showAttrPath(state, env, attrPath))
             : nullptr;
@@ -1585,7 +1591,7 @@ void EvalState::callFunction(Value & fun, std::span<Value *> args, Value & vRes,
             try {
                 auto dts = debugRepl
                     ? makeDebugTraceStacker(
-                        *this, *lambda.body, env2, positions[lambda.pos],
+                        *this, *lambda.body, env2, lambda.pos,
                         "while calling %s",
                         lambda.name
                         ? concatStrings("'", symbols[lambda.name], "'")
@@ -1720,9 +1726,7 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v)
             state,
             *this,
             env,
-            getPos()
-                ? std::make_shared<Pos>(state.positions[getPos()])
-                : nullptr,
+            getPos(),
             "while calling a function"
         )
         : nullptr;
@@ -2106,7 +2110,7 @@ void EvalState::forceValueDeep(Value & v)
                 try {
                     // If the value is a thunk, we're evaling. Otherwise no trace necessary.
                     auto dts = debugRepl && i.value->isThunk()
-                        ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, positions[i.pos],
+                        ? makeDebugTraceStacker(*this, *i.value->payload.thunk.expr, *i.value->payload.thunk.env, i.pos,
                             "while evaluating the attribute '%1%'", symbols[i.name])
                         : nullptr;
 
@@ -2384,7 +2388,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
         : [&]() {
             auto dstPath = fetchToStore(
                 *store,
-                path.resolveSymlinks(),
+                path.resolveSymlinks(SymlinkResolution::Ancestors),
                 settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
                 path.baseName(),
                 ContentAddressMethod::Raw::NixArchive,
@@ -3070,8 +3074,11 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
     auto i = lookupPathResolved.find(value);
     if (i != lookupPathResolved.end()) return i->second;
 
-    auto finish = [&](SourcePath res) {
-        debug("resolved search path element '%s' to '%s'", value, res);
+    auto finish = [&](std::optional<SourcePath> res) {
+        if (res)
+            debug("resolved search path element '%s' to '%s'", value, *res);
+        else
+            debug("failed to resolve search path element '%s'", value);
         lookupPathResolved.emplace(value, res);
         return res;
     };
@@ -3114,7 +3121,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
             }
         }
 
-        if (path.pathExists())
+        if (path.resolveSymlinks().pathExists())
             return finish(std::move(path));
         else {
             logWarning({
@@ -3123,8 +3130,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
         }
     }
 
-    debug("failed to resolve search path element '%s'", value);
-    return std::nullopt;
+    return finish(std::nullopt);
 }
 
 

src/libexpr/eval.hh

@@ -171,11 +171,28 @@ struct RegexCache;
 std::shared_ptr<RegexCache> makeRegexCache();
 
 struct DebugTrace {
-    std::shared_ptr<Pos> pos;
+    /* WARNING: Converting PosIdx -> Pos should be done with extra care. This is
+       due to the fact that operator[] of PosTable is incredibly expensive. */
+    std::variant<Pos, PosIdx> pos;
     const Expr & expr;
     const Env & env;
     HintFmt hint;
     bool isError;
+
+    Pos getPos(const PosTable & table) const
+    {
+        return std::visit(
+            overloaded{
+                [&](PosIdx idx) {
+                    // Prefer direct pos, but if noPos then try the expr.
+                    if (!idx)
+                        idx = expr.getPos();
+                    return table[idx];
+                },
+                [&](Pos pos) { return pos; },
+            },
+            pos);
+    }
 };
 
 class EvalState : public std::enable_shared_from_this<EvalState>
@@ -820,6 +837,15 @@ public:
      */
     [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
 
+    /**
+     * Realise the given string with context, and return the string with outputs instead of downstream output placeholders.
+     * @param[in] str the string to realise
+     * @param[out] paths all referenced store paths will be added to this set
+     * @return the realised string
+     * @throw EvalError if the value is not a string, path or derivation (see `coerceToString`)
+     */
+    std::string realiseString(Value & str, StorePathSet * storePathsOutMaybe, bool isIFD = true, const PosIdx pos = noPos);
+
     /* Call the binary path filter predicate used builtins.path etc. */
     bool callPathFilter(
         Value * filterFun,

src/libexpr/fetchurl.nix

@@ -1,40 +1,72 @@
-{ system ? "" # obsolete
-, url
-, hash ? "" # an SRI hash
+{
+  system ? "", # obsolete
+  url,
+  hash ? "", # an SRI hash
 
-# Legacy hash specification
-, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
-, outputHash ?
-    if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
-, outputHashAlgo ?
-    if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+  # Legacy hash specification
+  md5 ? "",
+  sha1 ? "",
+  sha256 ? "",
+  sha512 ? "",
+  outputHash ?
+    if hash != "" then
+      hash
+    else if sha512 != "" then
+      sha512
+    else if sha1 != "" then
+      sha1
+    else if md5 != "" then
+      md5
+    else
+      sha256,
+  outputHashAlgo ?
+    if hash != "" then
+      ""
+    else if sha512 != "" then
+      "sha512"
+    else if sha1 != "" then
+      "sha1"
+    else if md5 != "" then
+      "md5"
+    else
+      "sha256",
 
-, executable ? false
-, unpack ? false
-, name ? baseNameOf (toString url)
-, impure ? false
+  executable ? false,
+  unpack ? false,
+  name ? baseNameOf (toString url),
+  impure ? false,
 }:
 
-derivation ({
-  builder = "builtin:fetchurl";
+derivation (
+  {
+    builder = "builtin:fetchurl";
 
-  # New-style output content requirements.
-  outputHashMode = if unpack || executable then "recursive" else "flat";
+    # New-style output content requirements.
+    outputHashMode = if unpack || executable then "recursive" else "flat";
 
-  inherit name url executable unpack;
+    inherit
+      name
+      url
+      executable
+      unpack
+      ;
 
-  system = "builtin";
+    system = "builtin";
 
-  # No need to double the amount of network traffic
-  preferLocalBuild = true;
+    # No need to double the amount of network traffic
+    preferLocalBuild = true;
 
-  # This attribute does nothing; it's here to avoid changing evaluation results.
-  impureEnvVars = [
-    "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
-  ];
+    # This attribute does nothing; it's here to avoid changing evaluation results.
+    impureEnvVars = [
+      "http_proxy"
+      "https_proxy"
+      "ftp_proxy"
+      "all_proxy"
+      "no_proxy"
+    ];
 
-  # To make "nix-prefetch-url" work.
-  urls = [ url ];
-} // (if impure
-      then { __impure = true; }
-      else { inherit outputHashAlgo outputHash; }))
+    # To make "nix-prefetch-url" work.
+    urls = [ url ];
+  }
+  // (if impure then { __impure = true; } else { inherit outputHashAlgo outputHash; })
+)

src/libexpr/imported-drv-to-derivation.nix

@@ -1,21 +1,27 @@
-attrs @ { drvPath, outputs, name, ... }:
+attrs@{
+  drvPath,
+  outputs,
+  name,
+  ...
+}:
 
 let
 
-  commonAttrs = (builtins.listToAttrs outputsList) //
-    { all = map (x: x.value) outputsList;
-      inherit drvPath name;
-      type = "derivation";
-    };
+  commonAttrs = (builtins.listToAttrs outputsList) // {
+    all = map (x: x.value) outputsList;
+    inherit drvPath name;
+    type = "derivation";
+  };
 
-  outputToAttrListElement = outputName:
-    { name = outputName;
-      value = commonAttrs // {
-        outPath = builtins.getAttr outputName attrs;
-        inherit outputName;
-      };
+  outputToAttrListElement = outputName: {
+    name = outputName;
+    value = commonAttrs // {
+      outPath = builtins.getAttr outputName attrs;
+      inherit outputName;
     };
-    
+  };
+
   outputsList = map outputToAttrListElement outputs;
-    
-in (builtins.head outputsList).value
+
+in
+(builtins.head outputsList).value

src/libexpr/meson.build

@@ -171,8 +171,6 @@ headers = [config_h] + files(
   # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',
-  'pos-idx.hh',
-  'pos-table.hh',
   'primops.hh',
   'print-ambiguous.hh',
   'print-options.hh',
@@ -188,6 +186,9 @@ headers = [config_h] + files(
 
 subdir('primops')
 
+subdir('nix-meson-build-support/export-all-symbols')
+subdir('nix-meson-build-support/windows-version')
+
 this_library = library(
   'nixexpr',
   sources,
@@ -195,6 +196,8 @@ this_library = library(
   lexer_tab,
   generated_headers,
   dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
   prelink : true, # For C++ static initializers
   install : true,
 )

src/libexpr/nixexpr.cc

@@ -601,41 +601,6 @@ void ExprLambda::setDocComment(DocComment docComment) {
     }
 };
 
-
-
-/* Position table. */
-
-Pos PosTable::operator[](PosIdx p) const
-{
-    auto origin = resolve(p);
-    if (!origin)
-        return {};
-
-    const auto offset = origin->offsetOf(p);
-
-    Pos result{0, 0, origin->origin};
-    auto lines = this->lines.lock();
-    auto linesForInput = (*lines)[origin->offset];
-
-    if (linesForInput.empty()) {
-        auto source = result.getSource().value_or("");
-        const char * begin = source.data();
-        for (Pos::LinesIterator it(source), end; it != end; it++)
-            linesForInput.push_back(it->data() - begin);
-        if (linesForInput.empty())
-            linesForInput.push_back(0);
-    }
-    // as above: the first line starts at byte 0 and is always present
-    auto lineStartOffset = std::prev(
-        std::upper_bound(linesForInput.begin(), linesForInput.end(), offset));
-
-    result.line = 1 + (lineStartOffset - linesForInput.begin());
-    result.column = 1 + (offset - *lineStartOffset);
-    return result;
-}
-
-
-
 /* Symbol table. */
 
 size_t SymbolTable::totalSize() const

src/libexpr/package.nix

@@ -1,33 +1,34 @@
-{ lib
-, stdenv
-, mkMesonLibrary
+{
+  lib,
+  stdenv,
+  mkMesonLibrary,
 
-, bison
-, flex
-, cmake # for resolving toml11 dep
+  bison,
+  flex,
+  cmake, # for resolving toml11 dep
 
-, nix-util
-, nix-store
-, nix-fetchers
-, boost
-, boehmgc
-, nlohmann_json
-, toml11
+  nix-util,
+  nix-store,
+  nix-fetchers,
+  boost,
+  boehmgc,
+  nlohmann_json,
+  toml11,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 
-# Whether to use garbage collection for the Nix language evaluator.
-#
-# If it is disabled, we just leak memory, but this is not as bad as it
-# sounds so long as evaluation just takes places within short-lived
-# processes. (When the process exits, the memory is reclaimed; it is
-# only leaked *within* the process.)
-#
-# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
-# symbol is missing during linking.
-, enableGC ? !stdenv.hostPlatform.isWindows
+  # Whether to use garbage collection for the Nix language evaluator.
+  #
+  # If it is disabled, we just leak memory, but this is not as bad as it
+  # sounds so long as evaluation just takes places within short-lived
+  # processes. (When the process exits, the memory is reclaimed; it is
+  # only leaked *within* the process.)
+  #
+  # Temporarily disabled on Windows because the `GC_throw_bad_alloc`
+  # symbol is missing during linking.
+  enableGC ? !stdenv.hostPlatform.isWindows,
 }:
 
 let
@@ -51,10 +52,7 @@ mkMesonLibrary (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "hh") ./.)
     ./lexer.l
     ./parser.y
-    (fileset.difference
-      (fileset.fileFilter (file: file.hasExt "nix") ./.)
-      ./package.nix
-    )
+    (fileset.difference (fileset.fileFilter (file: file.hasExt "nix") ./.) ./package.nix)
   ];
 
   nativeBuildInputs = [
@@ -79,25 +77,10 @@ mkMesonLibrary (finalAttrs: {
     nlohmann_json
   ] ++ lib.optional enableGC boehmgc;
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
     (lib.mesonEnable "gc" enableGC)
   ];
 
-  env = {
-    # Needed for Meson to find Boost.
-    # https://github.com/NixOS/nixpkgs/issues/86131.
-    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
-    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
-  };
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libexpr/primops.cc

@@ -47,6 +47,15 @@ static inline Value * mkString(EvalState & state, const std::csub_match & match)
     return v;
 }
 
+std::string EvalState::realiseString(Value & s, StorePathSet * storePathsOutMaybe, bool isIFD, const PosIdx pos)
+{
+    nix::NixStringContext stringContext;
+    auto rawStr = coerceToString(pos, s, stringContext, "while realising a string").toOwned();
+    auto rewrites = realiseContext(stringContext, storePathsOutMaybe, isIFD);
+
+    return nix::rewriteStrings(rawStr, rewrites);
+}
+
 StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD)
 {
     std::vector<DerivedPath::Built> drvs;

src/libexpr/primops/derivation.nix

@@ -26,27 +26,34 @@
   Note that `derivation` is very bare-bones, and provides almost no commands during the build.
   Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment.
 */
-drvAttrs @ { outputs ? [ "out" ], ... }:
+drvAttrs@{
+  outputs ? [ "out" ],
+  ...
+}:
 
 let
 
   strict = derivationStrict drvAttrs;
 
-  commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) //
-    { all = map (x: x.value) outputsList;
+  commonAttrs =
+    drvAttrs
+    // (builtins.listToAttrs outputsList)
+    // {
+      all = map (x: x.value) outputsList;
       inherit drvAttrs;
     };
 
-  outputToAttrListElement = outputName:
-    { name = outputName;
-      value = commonAttrs // {
-        outPath = builtins.getAttr outputName strict;
-        drvPath = strict.drvPath;
-        type = "derivation";
-        inherit outputName;
-      };
+  outputToAttrListElement = outputName: {
+    name = outputName;
+    value = commonAttrs // {
+      outPath = builtins.getAttr outputName strict;
+      drvPath = strict.drvPath;
+      type = "derivation";
+      inherit outputName;
     };
+  };
 
   outputsList = map outputToAttrListElement outputs;
 
-in (builtins.head outputsList).value
+in
+(builtins.head outputsList).value

src/libfetchers-tests/package.nix

@@ -1,19 +1,20 @@
-{ lib
-, buildPackages
-, stdenv
-, mkMesonExecutable
+{
+  lib,
+  buildPackages,
+  stdenv,
+  mkMesonExecutable,
 
-, nix-fetchers
-, nix-store-test-support
+  nix-fetchers,
+  nix-store-test-support,
 
-, rapidcheck
-, gtest
-, runCommand
+  rapidcheck,
+  gtest,
+  runCommand,
 
-# Configuration Options
+  # Configuration Options
 
-, version
-, resolvePath
+  version,
+  resolvePath,
 }:
 
 let
@@ -43,29 +44,27 @@ mkMesonExecutable (finalAttrs: {
     gtest
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 
   passthru = {
     tests = {
-      run = runCommand "${finalAttrs.pname}-run" {
-        meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
-      } (lib.optionalString stdenv.hostPlatform.isWindows ''
-        export HOME="$PWD/home-dir"
-        mkdir -p "$HOME"
-      '' + ''
-        export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
-        ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
-        touch $out
-      '');
+      run =
+        runCommand "${finalAttrs.pname}-run"
+          {
+            meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
+          }
+          (
+            lib.optionalString stdenv.hostPlatform.isWindows ''
+              export HOME="$PWD/home-dir"
+              mkdir -p "$HOME"
+            ''
+            + ''
+              export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
+              ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
+              touch $out
+            ''
+          );
     };
   };
 

src/libfetchers/fetch-to-store.cc

@@ -4,6 +4,21 @@
 
 namespace nix {
 
+fetchers::Cache::Key makeFetchToStoreCacheKey(
+    const std::string &name,
+    const std::string &fingerprint,
+    ContentAddressMethod method,
+    const std::string &path)
+{
+    return fetchers::Cache::Key{"fetchToStore", {
+        {"name", name},
+        {"fingerprint", fingerprint},
+        {"method", std::string{method.render()}},
+        {"path", path}
+    }};
+
+}
+
 StorePath fetchToStore(
     Store & store,
     const SourcePath & path,
@@ -19,12 +34,7 @@ StorePath fetchToStore(
     std::optional<fetchers::Cache::Key> cacheKey;
 
     if (!filter && path.accessor->fingerprint) {
-        cacheKey = fetchers::Cache::Key{"fetchToStore", {
-            {"name", std::string{name}},
-            {"fingerprint", *path.accessor->fingerprint},
-            {"method", std::string{method.render()}},
-            {"path", path.path.abs()}
-        }};
+        cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs());
         if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) {
             debug("store path cache hit for '%s'", path);
             return res->storePath;

src/libfetchers/fetch-to-store.hh

@@ -5,6 +5,7 @@
 #include "file-system.hh"
 #include "repair-flag.hh"
 #include "file-content-address.hh"
+#include "cache.hh"
 
 namespace nix {
 
@@ -22,4 +23,7 @@ StorePath fetchToStore(
     PathFilter * filter = nullptr,
     RepairFlag repair = NoRepair);
 
+fetchers::Cache::Key makeFetchToStoreCacheKey(
+    const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path);
+
 }

src/libfetchers/git-utils.cc

@@ -206,7 +206,8 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil
 
 } // extern "C"
 
-static void initRepoAtomically(std::filesystem::path &path, bool bare) {
+static void initRepoAtomically(std::filesystem::path &path, bool bare)
+{
     if (pathExists(path.string())) return;
 
     Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() }));
@@ -507,7 +508,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
      */
     ref<GitSourceAccessor> getRawAccessor(const Hash & rev);
 
-    ref<SourceAccessor> getAccessor(const Hash & rev, bool exportIgnore) override;
+    ref<SourceAccessor> getAccessor(
+        const Hash & rev,
+        bool exportIgnore,
+        std::string displayPrefix) override;
 
     ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override;
 
@@ -544,13 +548,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         //       then use code that was removed in this commit (see blame)
 
         auto dir = this->path;
-        Strings gitArgs;
-        if (shallow) {
-            gitArgs = { "-C", dir.string(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec };
-        }
-        else {
-            gitArgs = { "-C", dir.string(), "fetch", "--quiet", "--force", "--", url, refspec };
-        }
+        Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"};
+        if (shallow)
+            append(gitArgs, {"--depth", "1"});
+        append(gitArgs, {std::string("--"), url, refspec});
 
         runProgram(RunOptions {
             .program = "git",
@@ -629,7 +630,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     Hash treeHashToNarHash(const Hash & treeHash) override
     {
-        auto accessor = getAccessor(treeHash, false);
+        auto accessor = getAccessor(treeHash, false, "");
 
         fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}};
 
@@ -1196,16 +1197,18 @@ ref<GitSourceAccessor> GitRepoImpl::getRawAccessor(const Hash & rev)
     return make_ref<GitSourceAccessor>(self, rev);
 }
 
-ref<SourceAccessor> GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore)
+ref<SourceAccessor> GitRepoImpl::getAccessor(
+    const Hash & rev,
+    bool exportIgnore,
+    std::string displayPrefix)
 {
     auto self = ref<GitRepoImpl>(shared_from_this());
     ref<GitSourceAccessor> rawGitAccessor = getRawAccessor(rev);
-    if (exportIgnore) {
+    rawGitAccessor->setPathDisplay(std::move(displayPrefix));
+    if (exportIgnore)
         return make_ref<GitExportIgnoreSourceAccessor>(self, rawGitAccessor, rev);
-    }
-    else {
+    else
         return rawGitAccessor;
-    }
 }
 
 ref<SourceAccessor> GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError)
@@ -1238,7 +1241,7 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
 
-    auto accessor = getAccessor(rev, exportIgnore);
+    auto accessor = getAccessor(rev, exportIgnore, "");
     if (!accessor->pathExists(modulesFile)) return {};
 
     /* Parse it and get the revision of each submodule. */

src/libfetchers/git-utils.hh

@@ -86,7 +86,10 @@ struct GitRepo
 
     virtual bool hasObject(const Hash & oid) = 0;
 
-    virtual ref<SourceAccessor> getAccessor(const Hash & rev, bool exportIgnore) = 0;
+    virtual ref<SourceAccessor> getAccessor(
+        const Hash & rev,
+        bool exportIgnore,
+        std::string displayPrefix) = 0;
 
     virtual ref<SourceAccessor> getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0;
 

src/libfetchers/git.cc

@@ -69,7 +69,7 @@ std::optional<std::string> readHead(const Path & path)
 
     std::string_view line = output;
     line = line.substr(0, line.find("\n"));
-    if (const auto parseResult = git::parseLsRemoteLine(line)) {
+    if (const auto parseResult = git::parseLsRemoteLine(line); parseResult && parseResult->reference == "HEAD") {
         switch (parseResult->kind) {
             case git::LsRemoteRefLine::Kind::Symbolic:
                 debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path);
@@ -459,8 +459,14 @@ struct GitInputScheme : InputScheme
                     url);
             }
             repoInfo.location = std::filesystem::absolute(url.path);
-        } else
+        } else {
+            if (url.scheme == "file")
+                /* Query parameters are meaningless for file://, but
+                   Git interprets them as part of the file name. So get
+                   rid of them. */
+                url.query.clear();
             repoInfo.location = url;
+        }
 
         // If this is a local directory and no ref or revision is
         // given, then allow the use of an unclean working tree.
@@ -605,16 +611,16 @@ struct GitInputScheme : InputScheme
                 try {
                     auto fetchRef =
                         getAllRefsAttr(input)
-                        ? "refs/*"
+                        ? "refs/*:refs/*"
                         : input.getRev()
                         ? input.getRev()->gitRev()
                         : ref.compare(0, 5, "refs/") == 0
-                        ? ref
+                        ? fmt("%1%:%1%", ref)
                         : ref == "HEAD"
                         ? ref
-                        : "refs/heads/" + ref;
+                        : fmt("%1%:%1%", "refs/heads/" + ref);
 
-                    repo->fetch(repoUrl.to_string(), fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
+                    repo->fetch(repoUrl.to_string(), fetchRef, getShallowAttr(input));
                 } catch (Error & e) {
                     if (!std::filesystem::exists(localRefFile)) throw;
                     logError(e.info());
@@ -672,9 +678,7 @@ struct GitInputScheme : InputScheme
         verifyCommit(input, repo);
 
         bool exportIgnore = getExportIgnoreAttr(input);
-        auto accessor = repo->getAccessor(rev, exportIgnore);
-
-        accessor->setPathDisplay("«" + input.to_string() + "»");
+        auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string() + "»");
 
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
@@ -737,8 +741,6 @@ struct GitInputScheme : InputScheme
                 exportIgnore,
                 makeNotAllowedError(repoInfo.locationToArg()));
 
-        accessor->setPathDisplay(repoInfo.locationToArg());
-
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */

src/libfetchers/github.cc

@@ -294,9 +294,10 @@ struct GitArchiveInputScheme : InputScheme
         #endif
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
-        auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false);
-
-        accessor->setPathDisplay("«" + input.to_string() + "»");
+        auto accessor = getTarballCache()->getAccessor(
+            tarballInfo.treeHash,
+            false,
+            "«" + input.to_string() + "»");
 
         return {accessor, input};
     }

src/libfetchers/meson.build

@@ -76,10 +76,15 @@ headers = files(
   'tarball.hh',
 )
 
+subdir('nix-meson-build-support/export-all-symbols')
+subdir('nix-meson-build-support/windows-version')
+
 this_library = library(
   'nixfetchers',
   sources,
   dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
   prelink : true, # For C++ static initializers
   install : true,
 )

src/libfetchers/package.nix

@@ -1,14 +1,15 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util
-, nix-store
-, nlohmann_json
-, libgit2
+  nix-util,
+  nix-store,
+  nlohmann_json,
+  libgit2,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -40,14 +41,6 @@ mkMesonLibrary (finalAttrs: {
     nlohmann_json
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libfetchers/path.cc

@@ -2,6 +2,8 @@
 #include "store-api.hh"
 #include "archive.hh"
 #include "store-path-accessor.hh"
+#include "fetch-to-store.hh"
+#include "cache.hh"
 
 namespace nix::fetchers {
 
@@ -125,7 +127,7 @@ struct PathInputScheme : InputScheme
 
         auto absPath = getAbsPath(input);
 
-        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to the store", absPath));
+        Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
 
         // FIXME: check whether access to 'path' is allowed.
         auto storePath = store->maybeParseStorePath(absPath.string());
@@ -142,6 +144,14 @@ struct PathInputScheme : InputScheme
             storePath = store->addToStoreFromDump(*src, "source");
         }
 
+        // To avoid copying the path again to the /nix/store, we need to add a cache entry.
+        ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
+        auto fp = getFingerprint(store, input);
+        if (fp) {
+            auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/");
+            fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath);
+        }
+
         /* Trust the lastModified value supplied by the user, if
            any. It's not a "secure" attribute so we don't care. */
         if (!input.getLastModified())

src/libfetchers/tarball.cc

@@ -105,7 +105,8 @@ DownloadFileResult downloadFile(
 
 static DownloadTarballResult downloadTarball_(
     const std::string & url,
-    const Headers & headers)
+    const Headers & headers,
+    const std::string & displayPrefix)
 {
     Cache::Key cacheKey{"tarball", {{"url", url}}};
 
@@ -118,7 +119,7 @@ static DownloadTarballResult downloadTarball_(
             .treeHash = treeHash,
             .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
             .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
-            .accessor = getTarballCache()->getAccessor(treeHash, false),
+            .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix),
         };
     };
 
@@ -371,9 +372,10 @@ struct TarballInputScheme : CurlInputScheme
     {
         auto input(_input);
 
-        auto result = downloadTarball_(getStrAttr(input.attrs, "url"), {});
-
-        result.accessor->setPathDisplay("«" + input.to_string() + "»");
+        auto result = downloadTarball_(
+            getStrAttr(input.attrs, "url"),
+            {},
+            "«" + input.to_string() + "»");
 
         if (result.immutableUrl) {
             auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl);

src/libflake-c/package.nix

@@ -1,13 +1,14 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-store-c
-, nix-expr-c
-, nix-flake
+  nix-store-c,
+  nix-expr-c,
+  nix-flake,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -37,14 +38,6 @@ mkMesonLibrary (finalAttrs: {
     nix-flake
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libflake-tests/package.nix

@@ -1,20 +1,21 @@
-{ lib
-, buildPackages
-, stdenv
-, mkMesonExecutable
+{
+  lib,
+  buildPackages,
+  stdenv,
+  mkMesonExecutable,
 
-, nix-flake
-, nix-flake-c
-, nix-expr-test-support
+  nix-flake,
+  nix-flake-c,
+  nix-expr-test-support,
 
-, rapidcheck
-, gtest
-, runCommand
+  rapidcheck,
+  gtest,
+  runCommand,
 
-# Configuration Options
+  # Configuration Options
 
-, version
-, resolvePath
+  version,
+  resolvePath,
 }:
 
 let
@@ -45,30 +46,28 @@ mkMesonExecutable (finalAttrs: {
     gtest
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 
   passthru = {
     tests = {
-      run = runCommand "${finalAttrs.pname}-run" {
-        meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
-      } (lib.optionalString stdenv.hostPlatform.isWindows ''
-        export HOME="$PWD/home-dir"
-        mkdir -p "$HOME"
-      '' + ''
-        export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
-        export NIX_CONFIG="extra-experimental-features = flakes"
-        ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
-        touch $out
-      '');
+      run =
+        runCommand "${finalAttrs.pname}-run"
+          {
+            meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
+          }
+          (
+            lib.optionalString stdenv.hostPlatform.isWindows ''
+              export HOME="$PWD/home-dir"
+              mkdir -p "$HOME"
+            ''
+            + ''
+              export _NIX_TEST_UNIT_DATA=${resolvePath ./data}
+              export NIX_CONFIG="extra-experimental-features = flakes"
+              ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
+              touch $out
+            ''
+          );
     };
   };
 

src/libflake/flake/flake.cc

@@ -554,12 +554,12 @@ LockedFlake lockFlake(
 
                     /* Get the input flake, resolve 'path:./...'
                        flakerefs relative to the parent flake. */
-                    auto getInputFlake = [&]()
+                    auto getInputFlake = [&](const FlakeRef & ref)
                     {
                         if (auto resolvedPath = resolveRelativePath()) {
-                            return readFlake(state, *input.ref, *input.ref, *input.ref, *resolvedPath, inputPath);
+                            return readFlake(state, ref, ref, ref, *resolvedPath, inputPath);
                         } else {
-                            return getFlake(state, *input.ref, useRegistries, flakeCache, inputPath);
+                            return getFlake(state, ref, useRegistries, flakeCache, inputPath);
                         }
                     };
 
@@ -640,7 +640,7 @@ LockedFlake lockFlake(
                         }
 
                         if (mustRefetch) {
-                            auto inputFlake = getInputFlake();
+                            auto inputFlake = getInputFlake(oldLock->lockedRef);
                             nodePaths.emplace(childNode, inputFlake.path.parent());
                             computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, followsPrefix,
                                 inputFlake.path, false);
@@ -668,7 +668,7 @@ LockedFlake lockFlake(
                         auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref;
 
                         if (input.isFlake) {
-                            auto inputFlake = getInputFlake();
+                            auto inputFlake = getInputFlake(*input.ref);
 
                             auto childNode = make_ref<LockedNode>(
                                 inputFlake.lockedRef,

src/libflake/flake/flakeref.cc

@@ -39,7 +39,7 @@ FlakeRef FlakeRef::resolve(
     ref<Store> store,
     const fetchers::RegistryFilter & filter) const
 {
-    auto [input2, extraAttrs] = lookupInRegistries(store, input);
+    auto [input2, extraAttrs] = lookupInRegistries(store, input, filter);
     return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
 }
 

src/libflake/meson.build

@@ -58,10 +58,15 @@ headers = files(
   'flake/url-name.hh',
 )
 
+subdir('nix-meson-build-support/export-all-symbols')
+subdir('nix-meson-build-support/windows-version')
+
 this_library = library(
   'nixflake',
   sources,
   dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
   prelink : true, # For C++ static initializers
   install : true,
 )

src/libflake/package.nix

@@ -1,15 +1,16 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util
-, nix-store
-, nix-fetchers
-, nix-expr
-, nlohmann_json
+  nix-util,
+  nix-store,
+  nix-fetchers,
+  nix-expr,
+  nlohmann_json,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -39,14 +40,6 @@ mkMesonLibrary (finalAttrs: {
     nlohmann_json
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain-c/package.nix

@@ -1,14 +1,15 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util-c
-, nix-store
-, nix-store-c
-, nix-main
+  nix-util-c,
+  nix-store,
+  nix-store-c,
+  nix-main,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -39,14 +40,6 @@ mkMesonLibrary (finalAttrs: {
     nix-main
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libmain/meson.build

@@ -82,10 +82,15 @@ headers = [config_h] + files(
   'shared.hh',
 )
 
+subdir('nix-meson-build-support/export-all-symbols')
+subdir('nix-meson-build-support/windows-version')
+
 this_library = library(
   'nixmain',
   sources,
   dependencies : deps_public + deps_private + deps_other,
+  include_directories : include_dirs,
+  link_args: linker_export_flags,
   prelink : true, # For C++ static initializers
   install : true,
 )

src/libmain/package.nix

@@ -1,14 +1,15 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, openssl
+  openssl,
 
-, nix-util
-, nix-store
+  nix-util,
+  nix-store,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -36,14 +37,6 @@ mkMesonLibrary (finalAttrs: {
     openssl
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };

src/libmain/shared.cc

@@ -315,20 +315,6 @@ void printVersion(const std::string & programName)
     throw Exit();
 }
 
-
-void showManPage(const std::string & name)
-{
-    restoreProcessContext();
-    setEnv("MANPATH", settings.nixManDir.c_str());
-    execlp("man", "man", name.c_str(), nullptr);
-    if (errno == ENOENT) {
-        // Not SysError because we don't want to suffix the errno, aka No such file or directory.
-        throw Error("The '%1%' command was not found, but it is needed for '%2%' and some other '%3%' commands' help text. Perhaps you could install the '%1%' command?", "man", name.c_str(), "nix-*");
-    }
-    throw SysError("command 'man %1%' failed", name.c_str());
-}
-
-
 int handleExceptions(const std::string & programName, std::function<void()> fun)
 {
     ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this

src/libmain/shared.hh

@@ -70,11 +70,6 @@ struct LegacyArgs : public MixCommonArgs, public RootArgs
 };
 
 
-/**
- * Show the manual page for the specified program.
- */
-void showManPage(const std::string & name);
-
 /**
  * The constructor of this class starts a pager if standard output is a
  * terminal and $PAGER is set. Standard output is redirected to the

src/libstore-c/package.nix

@@ -1,12 +1,13 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util-c
-, nix-store
+  nix-util-c,
+  nix-store,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -35,14 +36,6 @@ mkMesonLibrary (finalAttrs: {
     nix-store
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libstore-test-support/package.nix

@@ -1,15 +1,16 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util-test-support
-, nix-store
-, nix-store-c
+  nix-util-test-support,
+  nix-store,
+  nix-store-c,
 
-, rapidcheck
+  rapidcheck,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -39,14 +40,6 @@ mkMesonLibrary (finalAttrs: {
     rapidcheck
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libstore-tests/package.nix

@@ -1,21 +1,22 @@
-{ lib
-, buildPackages
-, stdenv
-, mkMesonExecutable
+{
+  lib,
+  buildPackages,
+  stdenv,
+  mkMesonExecutable,
 
-, nix-store
-, nix-store-c
-, nix-store-test-support
-, sqlite
+  nix-store,
+  nix-store-c,
+  nix-store-test-support,
+  sqlite,
 
-, rapidcheck
-, gtest
-, runCommand
+  rapidcheck,
+  gtest,
+  runCommand,
 
-# Configuration Options
+  # Configuration Options
 
-, version
-, filesetToSource
+  version,
+  filesetToSource,
 }:
 
 let
@@ -51,39 +52,38 @@ mkMesonExecutable (finalAttrs: {
     nix-store-test-support
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 
   passthru = {
     tests = {
-      run = let
-        # Some data is shared with the functional tests: they create it,
-        # we consume it.
-        data = filesetToSource {
-          root = ../..;
-          fileset = lib.fileset.unions [
-            ./data
-            ../../tests/functional/derivation
-          ];
-        };
-      in runCommand "${finalAttrs.pname}-run" {
-        meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
-      } (lib.optionalString stdenv.hostPlatform.isWindows ''
-        export HOME="$PWD/home-dir"
-        mkdir -p "$HOME"
-      '' + ''
-        export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"}
-        ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
-        touch $out
-      '');
+      run =
+        let
+          # Some data is shared with the functional tests: they create it,
+          # we consume it.
+          data = filesetToSource {
+            root = ../..;
+            fileset = lib.fileset.unions [
+              ./data
+              ../../tests/functional/derivation
+            ];
+          };
+        in
+        runCommand "${finalAttrs.pname}-run"
+          {
+            meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
+          }
+          (
+            lib.optionalString stdenv.hostPlatform.isWindows ''
+              export HOME="$PWD/home-dir"
+              mkdir -p "$HOME"
+            ''
+            + ''
+              export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"}
+              ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
+              touch $out
+            ''
+          );
     };
   };
 

src/libstore/builtins/unpack-channel.cc

@@ -23,11 +23,7 @@ void builtinUnpackChannel(
         throw Error("channelName is not allowed to contain filesystem separators, got %1%", channelName);
     }
 
-    try {
-        fs::create_directories(out);
-    } catch (fs::filesystem_error &) {
-        throw SysError("creating directory '%1%'", out.string());
-    }
+    createDirs(out);
 
     unpackTarfile(src, out);
 

src/libstore/daemon.cc

@@ -1025,6 +1025,7 @@ void processConnection(
 {
 #ifndef _WIN32 // TODO need graceful async exit support on Windows?
     auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr;
+    (void) monitor; // suppress warning
 #endif
 
     /* Exchange the greeting. */

src/libstore/filetransfer.cc

@@ -300,6 +300,14 @@ struct curlFileTransfer : public FileTransfer
             return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
+        #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+        static int cloexec_callback(void *, curl_socket_t curlfd, curlsocktype purpose) {
+            unix::closeOnExec(curlfd);
+            vomit("cloexec set for fd %i", curlfd);
+            return CURL_SOCKOPT_OK;
+        }
+        #endif
+
         void init()
         {
             if (!req) req = curl_easy_init();
@@ -359,6 +367,10 @@ struct curlFileTransfer : public FileTransfer
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
+            #if !defined(_WIN32) && LIBCURL_VERSION_NUM >= 0x071000
+            curl_easy_setopt(req, CURLOPT_SOCKOPTFUNCTION, cloexec_callback);
+            #endif
+
             curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);

src/libstore/globals.cc

@@ -65,7 +65,6 @@ Settings::Settings()
     , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
     , nixUserConfFiles(getUserConfigFiles())
-    , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
 #ifndef _WIN32
@@ -243,7 +242,7 @@ Path Settings::getDefaultSSLCertFile()
     return "";
 }
 
-const std::string nixVersion = PACKAGE_VERSION;
+std::string nixVersion = PACKAGE_VERSION;
 
 NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
     {SandboxMode::smEnabled, true},

src/libstore/globals.hh

@@ -84,11 +84,6 @@ public:
      */
     std::vector<Path> nixUserConfFiles;
 
-    /**
-     * The directory where the man pages are stored.
-     */
-    Path nixManDir;
-
     /**
      * File name of the socket the daemon listens to.
      */
@@ -1253,7 +1248,15 @@ void loadConfFile(AbstractConfig & config);
 // Used by the Settings constructor
 std::vector<Path> getUserConfigFiles();
 
-extern const std::string nixVersion;
+/**
+ * The version of Nix itself.
+ *
+ * This is not `const`, so that the Nix CLI can provide a more detailed version
+ * number including the git revision, without having to "re-compile" the entire
+ * set of Nix libraries to include that version, even when those libraries are
+ * not affected by the change.
+ */
+extern std::string nixVersion;
 
 /**
  * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests.

src/libstore/legacy-ssh-store.cc

@@ -69,7 +69,10 @@ ref<LegacySSHStore::Connection> LegacySSHStore::openConnection()
         command.push_back("--store");
         command.push_back(remoteStore.get());
     }
-    conn->sshConn = master.startCommand(std::move(command));
+    conn->sshConn = master.startCommand(std::move(command), std::list{extraSshArgs});
+    if (connPipeSize) {
+        conn->sshConn->trySetBufferSize(*connPipeSize);
+    }
     conn->to = FdSink(conn->sshConn->in.get());
     conn->from = FdSource(conn->sshConn->out.get());
 
@@ -100,19 +103,31 @@ std::string LegacySSHStore::getUri()
     return *uriSchemes().begin() + "://" + host;
 }
 
+std::map<StorePath, UnkeyedValidPathInfo> LegacySSHStore::queryPathInfosUncached(
+    const StorePathSet & paths)
+{
+    auto conn(connections->get());
+
+    /* No longer support missing NAR hash */
+    assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
+
+    debug("querying remote host '%s' for info on '%s'", host, concatStringsSep(", ", printStorePathSet(paths)));
+
+    auto infos = conn->queryPathInfos(*this, paths);
+
+    for (const auto & [_, info] : infos) {
+        if (info.narHash == Hash::dummy)
+            throw Error("NAR hash is now mandatory");
+    }
+
+    return infos;
+}
 
 void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
     Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
 {
     try {
-        auto conn(connections->get());
-
-        /* No longer support missing NAR hash */
-        assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4);
-
-        debug("querying remote host '%s' for info on '%s'", host, printStorePath(path));
-
-        auto infos = conn->queryPathInfos(*this, {path});
+        auto infos = queryPathInfosUncached({path});
 
         switch (infos.size()) {
         case 0:
@@ -120,9 +135,6 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path,
         case 1: {
             auto & [path2, info] = *infos.begin();
 
-            if (info.narHash == Hash::dummy)
-                throw Error("NAR hash is now mandatory");
-
             assert(path == path2);
             return callback(std::make_shared<ValidPathInfo>(
                 std::move(path),
@@ -193,13 +205,19 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source,
 
 void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink)
 {
-    auto conn(connections->get());
-    conn->narFromPath(*this, path, [&](auto & source) {
+    narFromPath(path, [&](auto & source) {
         copyNAR(source, sink);
     });
 }
 
 
+void LegacySSHStore::narFromPath(const StorePath & path, std::function<void(Source &)> fun)
+{
+    auto conn(connections->get());
+    conn->narFromPath(*this, path, fun);
+}
+
+
 static ServeProto::BuildOptions buildSettings()
 {
     return {
@@ -223,6 +241,19 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas
     return conn->getBuildDerivationResponse(*this);
 }
 
+std::function<BuildResult()> LegacySSHStore::buildDerivationAsync(
+    const StorePath & drvPath, const BasicDerivation & drv,
+    const ServeProto::BuildOptions & options)
+{
+    // Until we have C++23 std::move_only_function
+    auto conn = std::make_shared<Pool<Connection>::Handle>(connections->get());
+    (*conn)->putBuildDerivationRequest(*this, drvPath, drv, options);
+
+    return [this,conn]() -> BuildResult {
+        return (*conn)->getBuildDerivationResponse(*this);
+    };
+}
+
 
 void LegacySSHStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore)
 {
@@ -294,6 +325,32 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
 }
 
 
+StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths,
+    bool lock, SubstituteFlag maybeSubstitute)
+{
+    auto conn(connections->get());
+    return conn->queryValidPaths(*this,
+        lock, paths, maybeSubstitute);
+}
+
+
+void LegacySSHStore::addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths)
+{
+    auto conn(connections->get());
+    conn->to << ServeProto::Command::ImportPaths;
+    try {
+        srcStore.exportPaths(paths, conn->to);
+    } catch (...) {
+        conn->good = false;
+        throw;
+    }
+    conn->to.flush();
+
+    if (readInt(conn->from) != 1)
+        throw Error("remote machine failed to import closure");
+}
+
+
 void LegacySSHStore::connect()
 {
     auto conn(connections->get());
@@ -307,6 +364,28 @@ unsigned int LegacySSHStore::getProtocol()
 }
 
 
+pid_t LegacySSHStore::getConnectionPid()
+{
+    auto conn(connections->get());
+#ifndef _WIN32
+    return conn->sshConn->sshPid;
+#else
+    // TODO: Implement
+    return 0;
+#endif
+}
+
+
+LegacySSHStore::ConnectionStats LegacySSHStore::getConnectionStats()
+{
+    auto conn(connections->get());
+    return {
+        .bytesReceived = conn->from.read,
+        .bytesSent = conn->to.written,
+    };
+}
+
+
 /**
  * The legacy ssh protocol doesn't support checking for trusted-user.
  * Try using ssh-ng:// instead if you want to know.

src/libstore/legacy-ssh-store.hh

@@ -6,6 +6,7 @@
 #include "ssh.hh"
 #include "callback.hh"
 #include "pool.hh"
+#include "serve-protocol.hh"
 
 namespace nix {
 
@@ -24,6 +25,16 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig
     const Setting<int> maxConnections{this, 1, "max-connections",
         "Maximum number of concurrent SSH connections."};
 
+    /**
+     * Hack for hydra
+     */
+    Strings extraSshArgs = {};
+
+    /**
+     * Exposed for hydra
+     */
+    std::optional<size_t> connPipeSize;
+
     const std::string name() override { return "SSH Store"; }
 
     static std::set<std::string> uriSchemes() { return {"ssh"}; }
@@ -60,11 +71,24 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
     void queryPathInfoUncached(const StorePath & path,
         Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
 
+    std::map<StorePath, UnkeyedValidPathInfo> queryPathInfosUncached(
+        const StorePathSet & paths);
+
     void addToStore(const ValidPathInfo & info, Source & source,
         RepairFlag repair, CheckSigsFlag checkSigs) override;
 
     void narFromPath(const StorePath & path, Sink & sink) override;
 
+    /**
+     * Hands over the connection temporarily as source to the given
+     * function. The function must not consume beyond the NAR; it can
+     * not just blindly try to always read more bytes until it is
+     * cut-off.
+     *
+     * This is exposed for sake of Hydra.
+     */
+    void narFromPath(const StorePath & path, std::function<void(Source &)> fun);
+
     std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
     { unsupported("queryPathFromHashPart"); }
 
@@ -93,6 +117,16 @@ public:
     BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
         BuildMode buildMode) override;
 
+    /**
+     * Note, the returned function must only be called once, or we'll
+     * try to read from the connection twice.
+     *
+     * @todo Use C++23 `std::move_only_function`.
+     */
+    std::function<BuildResult()> buildDerivationAsync(
+        const StorePath & drvPath, const BasicDerivation & drv,
+        const ServeProto::BuildOptions & options);
+
     void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override;
 
     void ensurePath(const StorePath & path) override
@@ -119,10 +153,36 @@ public:
     StorePathSet queryValidPaths(const StorePathSet & paths,
         SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
+    /**
+     * Custom variation that atomically creates temp locks on the remote
+     * side.
+     *
+     * This exists to prevent a race where the remote host
+     * garbage-collects paths that are already there. Optionally, ask
+     * the remote host to substitute missing paths.
+     */
+    StorePathSet queryValidPaths(const StorePathSet & paths,
+        bool lock,
+        SubstituteFlag maybeSubstitute = NoSubstitute);
+
+    /**
+     * Just exists because this is exactly what Hydra was doing, and we
+     * don't yet want an algorithmic change.
+     */
+    void addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths);
+
     void connect() override;
 
     unsigned int getProtocol() override;
 
+    struct ConnectionStats {
+        size_t bytesReceived, bytesSent;
+    };
+
+    ConnectionStats getConnectionStats();
+
+    pid_t getConnectionPid();
+
     /**
      * The legacy ssh protocol doesn't support checking for trusted-user.
      * Try using ssh-ng:// instead if you want to know.

src/libstore/local-store.cc

@@ -116,7 +116,7 @@ LocalStore::LocalStore(
     state->stmts = std::make_unique<State::Stmts>();
 
     /* Create missing state directories if they don't already exist. */
-    createDirs(realStoreDir);
+    createDirs(realStoreDir.get());
     if (readOnly) {
         experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore);
     } else {
@@ -136,7 +136,12 @@ LocalStore::LocalStore(
     for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) {
         createDirs(perUserDir);
         if (!readOnly) {
-            if (chmod(perUserDir.c_str(), 0755) == -1)
+            auto st = lstat(perUserDir);
+
+            // Skip chmod call if the directory already has the correct permissions (0755).
+            // This is to avoid failing when the executing user lacks permissions to change the directory's permissions
+            // even if it would be no-op.
+            if ((st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != 0755 && chmod(perUserDir.c_str(), 0755) == -1)
                 throw SysError("could not set permissions on '%s' to 755", perUserDir);
         }
     }
@@ -243,7 +248,7 @@ LocalStore::LocalStore(
     else if (curSchema == 0) { /* new store */
         curSchema = nixSchemaVersion;
         openDB(*state, true);
-        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true);
+        writeFile(schemaPath, fmt("%1%", curSchema), 0666, FsSync::Yes);
     }
 
     else if (curSchema < nixSchemaVersion) {
@@ -294,7 +299,7 @@ LocalStore::LocalStore(
             txn.commit();
         }
 
-        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true);
+        writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, FsSync::Yes);
 
         lockFile(globalLock.get(), ltRead, true);
     }

src/libstore/meson.build

@@ -112,7 +112,8 @@ deps_public += nlohmann_json
 sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19')
 deps_private += sqlite
 
-# AWS C++ SDK has bad pkg-config
+# AWS C++ SDK has bad pkg-config. See
+# https://github.com/aws/aws-sdk-cpp/issues/2673 for details.
 aws_s3 = dependency('aws-cpp-sdk-s3', required : false)
 configdata.set('ENABLE_S3', aws_s3.found().to_int())
 if aws_s3.found()

src/libstore/package.nix

@@ -1,25 +1,30 @@
-{ lib
-, stdenv
-, mkMesonLibrary
+{
+  lib,
+  stdenv,
+  mkMesonLibrary,
 
-, unixtools
-, darwin
+  unixtools,
+  darwin,
 
-, nix-util
-, boost
-, curl
-, aws-sdk-cpp
-, libseccomp
-, nlohmann_json
-, sqlite
+  nix-util,
+  boost,
+  curl,
+  aws-sdk-cpp,
+  libseccomp,
+  nlohmann_json,
+  sqlite,
 
-, busybox-sandbox-shell ? null
+  busybox-sandbox-shell ? null,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 
-, embeddedSandboxShell ? stdenv.hostPlatform.isStatic
+  embeddedSandboxShell ? stdenv.hostPlatform.isStatic,
+
+  withAWS ?
+    # Default is this way because there have been issues building this dependency
+    stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin),
 }:
 
 let
@@ -48,46 +53,32 @@ mkMesonLibrary (finalAttrs: {
     (fileset.fileFilter (file: file.hasExt "sql") ./.)
   ];
 
-  nativeBuildInputs =
-    lib.optional embeddedSandboxShell unixtools.hexdump;
+  nativeBuildInputs = lib.optional embeddedSandboxShell unixtools.hexdump;
 
-  buildInputs = [
-    boost
-    curl
-    sqlite
-  ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
+  buildInputs =
+    [
+      boost
+      curl
+      sqlite
+    ]
+    ++ lib.optional stdenv.hostPlatform.isLinux libseccomp
     # There have been issues building these dependencies
     ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
-    ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
-      aws-sdk-cpp
-  ;
+    ++ lib.optional withAWS aws-sdk-cpp;
 
   propagatedBuildInputs = [
     nix-util
     nlohmann_json
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
-  mesonFlags = [
-    (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
-    (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
-  ] ++ lib.optionals stdenv.hostPlatform.isLinux [
-    (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
-  ];
-
-  env = {
-    # Needed for Meson to find Boost.
-    # https://github.com/NixOS/nixpkgs/issues/86131.
-    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
-    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
-  };
+  mesonFlags =
+    [
+      (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux)
+      (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell)
+    ]
+    ++ lib.optionals stdenv.hostPlatform.isLinux [
+      (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox")
+    ];
 
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;

src/libstore/remote-store.cc

@@ -539,11 +539,21 @@ void RemoteStore::addMultipleToStore(
     RepairFlag repair,
     CheckSigsFlag checkSigs)
 {
+    // `addMultipleToStore` is single threaded
+    size_t bytesExpected = 0;
+    for (auto & [pathInfo, _] : pathsToCopy) {
+        bytesExpected += pathInfo.narSize;
+    }
+    act.setExpected(actCopyPath, bytesExpected);
+
     auto source = sinkToSource([&](Sink & sink) {
-        sink << pathsToCopy.size();
+        size_t nrTotal = pathsToCopy.size();
+        sink << nrTotal;
         // Reverse, so we can release memory at the original start
         std::reverse(pathsToCopy.begin(), pathsToCopy.end());
         while (!pathsToCopy.empty()) {
+            act.progress(nrTotal - pathsToCopy.size(), nrTotal, size_t(1), size_t(0));
+
             auto & [pathInfo, pathSource] = pathsToCopy.back();
             WorkerProto::Serialise<ValidPathInfo>::write(*this,
                  WorkerProto::WriteConn {

src/libstore/ssh.cc

@@ -240,4 +240,19 @@ Path SSHMaster::startMaster()
 
 #endif
 
+void SSHMaster::Connection::trySetBufferSize(size_t size)
+{
+#ifdef F_SETPIPE_SZ
+    /* This `fcntl` method of doing this takes a positive `int`. Check
+       and convert accordingly.
+
+       The function overall still takes `size_t` because this is more
+       portable for a platform-agnostic interface. */
+    assert(size <= INT_MAX);
+    int pipesize = size;
+    fcntl(in.get(), F_SETPIPE_SZ, pipesize);
+    fcntl(out.get(), F_SETPIPE_SZ, pipesize);
+#endif
+}
+
 }

src/libstore/ssh.hh

@@ -54,6 +54,18 @@ public:
         Pid sshPid;
 #endif
         AutoCloseFD out, in;
+
+        /**
+         * Try to set the buffer size in both directions to the
+         * designated amount, if possible. If not possible, does
+         * nothing.
+         *
+         * Current implementation is to use `fcntl` with `F_SETPIPE_SZ`,
+         * which is Linux-only. For this implementation, `size` must
+         * convertable to an `int`. In other words, it must be within
+         * `[0, INT_MAX]`.
+         */
+        void trySetBufferSize(size_t size);
     };
 
     /**

src/libstore/store-api.cc

@@ -230,20 +230,24 @@ void Store::addMultipleToStore(
 {
     std::atomic<size_t> nrDone{0};
     std::atomic<size_t> nrFailed{0};
-    std::atomic<uint64_t> bytesExpected{0};
     std::atomic<uint64_t> nrRunning{0};
 
     using PathWithInfo = std::pair<ValidPathInfo, std::unique_ptr<Source>>;
 
+    uint64_t bytesExpected = 0;
+
     std::map<StorePath, PathWithInfo *> infosMap;
     StorePathSet storePathsToAdd;
     for (auto & thingToAdd : pathsToCopy) {
+        bytesExpected += thingToAdd.first.narSize;
         infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd);
         storePathsToAdd.insert(thingToAdd.first.path);
     }
 
-    auto showProgress = [&]() {
-        act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
+    act.setExpected(actCopyPath, bytesExpected);
+
+    auto showProgress = [&, nrTotal = pathsToCopy.size()]() {
+        act.progress(nrDone, nrTotal, nrRunning, nrFailed);
     };
 
     processGraph<StorePath>(
@@ -259,9 +263,6 @@ void Store::addMultipleToStore(
                 return StorePathSet();
             }
 
-            bytesExpected += info.narSize;
-            act.setExpected(actCopyPath, bytesExpected);
-
             return info.references;
         },
 
@@ -1104,9 +1105,6 @@ std::map<StorePath, StorePath> copyPaths(
         return storePathForDst;
     };
 
-    // total is accessed by each copy, which are each handled in separate threads
-    std::atomic<uint64_t> total = 0;
-
     for (auto & missingPath : sortedMissing) {
         auto info = srcStore.queryPathInfo(missingPath);
 
@@ -1116,9 +1114,10 @@ std::map<StorePath, StorePath> copyPaths(
         ValidPathInfo infoForDst = *info;
         infoForDst.path = storePathForDst;
 
-        auto source = sinkToSource([&](Sink & sink) {
+        auto source = sinkToSource([&, narSize = info->narSize](Sink & sink) {
             // We can reasonably assume that the copy will happen whenever we
             // read the path, so log something about that at that point
+            uint64_t total = 0;
             auto srcUri = srcStore.getUri();
             auto dstUri = dstStore.getUri();
             auto storePathS = srcStore.printStorePath(missingPath);
@@ -1129,13 +1128,13 @@ std::map<StorePath, StorePath> copyPaths(
 
             LambdaSink progressSink([&](std::string_view data) {
                 total += data.size();
-                act.progress(total, info->narSize);
+                act.progress(total, narSize);
             });
             TeeSink tee { sink, progressSink };
 
             srcStore.narFromPath(missingPath, tee);
         });
-        pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
+        pathsToCopy.emplace_back(std::move(infoForDst), std::move(source));
     }
 
     dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);

src/libstore/unix/build/local-derivation-goal.cc

@@ -559,7 +559,14 @@ void LocalDerivationGoal::startBuilder()
     } else {
         tmpDir = topTmpDir;
     }
-    chownToBuilder(tmpDir);
+
+    /* The TOCTOU between the previous mkdir call and this open call is unavoidable due to
+       POSIX semantics.*/
+    tmpDirFd = AutoCloseFD{open(tmpDir.c_str(), O_RDONLY | O_NOFOLLOW | O_DIRECTORY)};
+    if (!tmpDirFd)
+        throw SysError("failed to open the build temporary directory descriptor '%1%'", tmpDir);
+
+    chownToBuilder(tmpDirFd.get(), tmpDir);
 
     for (auto & [outputName, status] : initialOutputs) {
         /* Set scratch path we'll actually use during the build.
@@ -1157,9 +1164,7 @@ void LocalDerivationGoal::initTmpDir()
             } else {
                 auto hash = hashString(HashAlgorithm::SHA256, i.first);
                 std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false);
-                Path p = tmpDir + "/" + fn;
-                writeFile(p, rewriteStrings(i.second, inputRewrites));
-                chownToBuilder(p);
+                writeBuilderFile(fn, rewriteStrings(i.second, inputRewrites));
                 env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
             }
         }
@@ -1264,11 +1269,9 @@ void LocalDerivationGoal::writeStructuredAttrs()
 
         auto jsonSh = writeStructuredAttrsShell(json);
 
-        writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
-        chownToBuilder(tmpDir + "/.attrs.sh");
+        writeBuilderFile(".attrs.sh", rewriteStrings(jsonSh, inputRewrites));
         env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh";
-        writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
-        chownToBuilder(tmpDir + "/.attrs.json");
+        writeBuilderFile(".attrs.json", rewriteStrings(json.dump(), inputRewrites));
         env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json";
     }
 }
@@ -1779,6 +1782,24 @@ void setupSeccomp()
 #endif
 }
 
+void LocalDerivationGoal::chownToBuilder(int fd, const Path & path)
+{
+    if (!buildUser) return;
+    if (fchown(fd, buildUser->getUID(), buildUser->getGID()) == -1)
+        throw SysError("cannot change ownership of file '%1%'", path);
+}
+
+void LocalDerivationGoal::writeBuilderFile(
+    const std::string & name,
+    std::string_view contents)
+{
+    auto path = std::filesystem::path(tmpDir) / name;
+    AutoCloseFD fd{openat(tmpDirFd.get(), name.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC | O_EXCL | O_NOFOLLOW, 0666)};
+    if (!fd)
+        throw SysError("creating file %s", path);
+    writeFile(fd, path, contents);
+    chownToBuilder(fd.get(), path);
+}
 
 void LocalDerivationGoal::runChild()
 {
@@ -2149,7 +2170,18 @@ void LocalDerivationGoal::runChild()
                without file-write* allowed, access() incorrectly returns EPERM
              */
             sandboxProfile += "(allow file-read* file-write* process-exec\n";
+
+            // We create multiple allow lists, to avoid exceeding a limit in the darwin sandbox interpreter.
+            // See https://github.com/NixOS/nix/issues/4119
+            // We split our allow groups approximately at half the actual limit, 1 << 16
+            const int breakpoint = sandboxProfile.length() + (1 << 14);
             for (auto & i : pathsInChroot) {
+
+                if (sandboxProfile.length() >= breakpoint) {
+                    debug("Sandbox break: %d %d", sandboxProfile.length(), breakpoint);
+                    sandboxProfile += ")\n(allow file-read* file-write* process-exec\n";
+                }
+
                 if (i.first != i.second.source)
                     throw Error(
                         "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
@@ -2657,10 +2689,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
                             wanted.to_string(HashFormat::SRI, true),
                             got.to_string(HashFormat::SRI, true)));
                 }
-                if (!newInfo0.references.empty())
+                if (!newInfo0.references.empty()) {
+                    auto numViolations = newInfo.references.size();
                     delayedException = std::make_exception_ptr(
-                        BuildError("illegal path references in fixed-output derivation '%s'",
-                            worker.store.printStorePath(drvPath)));
+                        BuildError("fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'",
+                            worker.store.printStorePath(drvPath),
+                            numViolations,
+                            worker.store.printStorePath(*newInfo.references.begin())));
+                }
 
                 return newInfo0;
             },
@@ -2927,8 +2963,12 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
                         spec.insert(worker.store.parseStorePath(i));
                     else if (auto output = get(outputs, i))
                         spec.insert(output->path);
-                    else
-                        throw BuildError("derivation contains an illegal reference specifier '%s'", i);
+                    else {
+                        std::string outputsListing = concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; });
+                        throw BuildError("derivation '%s' output check for '%s' contains an illegal reference specifier '%s',"
+                            " expected store path or output name (one of [%s])",
+                            worker.store.printStorePath(drvPath), outputName, i, outputsListing);
+                    }
                 }
 
                 auto used = recursive
@@ -3034,6 +3074,15 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
 void LocalDerivationGoal::deleteTmpDir(bool force)
 {
     if (topTmpDir != "") {
+        /* As an extra precaution, even in the event of `deletePath` failing to
+         * clean up, the `tmpDir` will be chowned as if we were to move
+         * it inside the Nix store.
+         *
+         * This hardens against an attack which smuggles a file descriptor
+         * to make use of the temporary directory.
+         */
+        chmod(topTmpDir.c_str(), 0000);
+
         /* Don't keep temporary directories for builtins because they
            might have privileged stuff (like a copy of netrc). */
         if (settings.keepFailed && !force && !drv->isBuiltin()) {

src/libstore/unix/build/local-derivation-goal.hh

@@ -37,6 +37,11 @@ struct LocalDerivationGoal : public DerivationGoal
      */
     Path topTmpDir;
 
+    /**
+     * The file descriptor of the temporary directory.
+     */
+    AutoCloseFD tmpDirFd;
+
     /**
      * The path of the temporary directory in the sandbox.
      */
@@ -244,9 +249,24 @@ struct LocalDerivationGoal : public DerivationGoal
 
     /**
      * Make a file owned by the builder.
+     *
+     * SAFETY: this function is prone to TOCTOU as it receives a path and not a descriptor.
+     * It's only safe to call in a child of a directory only visible to the owner.
      */
     void chownToBuilder(const Path & path);
 
+    /**
+     * Make a file owned by the builder addressed by its file descriptor.
+     */
+    void chownToBuilder(int fd, const Path & path);
+
+    /**
+     * Create a file in `tmpDir` owned by the builder.
+     */
+    void writeBuilderFile(
+        const std::string & name,
+        std::string_view contents);
+
     int getChildStatus() override;
 
     /**

src/libutil-c/package.nix

@@ -1,11 +1,12 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util
+  nix-util,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -33,14 +34,6 @@ mkMesonLibrary (finalAttrs: {
     nix-util
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libutil-test-support/package.nix

@@ -1,14 +1,15 @@
-{ lib
-, mkMesonLibrary
+{
+  lib,
+  mkMesonLibrary,
 
-, nix-util
-, nix-util-c
+  nix-util,
+  nix-util-c,
 
-, rapidcheck
+  rapidcheck,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -37,14 +38,6 @@ mkMesonLibrary (finalAttrs: {
     rapidcheck
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 

src/libutil-tests/monitorfdhup.cc

@@ -0,0 +1,18 @@
+#include "util.hh"
+#include "monitor-fd.hh"
+
+#include <sys/file.h>
+#include <gtest/gtest.h>
+
+namespace nix {
+TEST(MonitorFdHup, shouldNotBlock)
+{
+    Pipe p;
+    p.create();
+    {
+        // when monitor gets destroyed it should cancel the
+        // background thread and do not block
+        MonitorFdHup monitor(p.readSide.get());
+    }
+}
+}

src/libutil-tests/package.nix

@@ -1,19 +1,20 @@
-{ lib
-, buildPackages
-, stdenv
-, mkMesonExecutable
+{
+  lib,
+  buildPackages,
+  stdenv,
+  mkMesonExecutable,
 
-, nix-util
-, nix-util-c
-, nix-util-test-support
+  nix-util,
+  nix-util-c,
+  nix-util-test-support,
 
-, rapidcheck
-, gtest
-, runCommand
+  rapidcheck,
+  gtest,
+  runCommand,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -44,29 +45,27 @@ mkMesonExecutable (finalAttrs: {
     gtest
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
   ];
 
   passthru = {
     tests = {
-      run = runCommand "${finalAttrs.pname}-run" {
-        meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
-      } (lib.optionalString stdenv.hostPlatform.isWindows ''
-        export HOME="$PWD/home-dir"
-        mkdir -p "$HOME"
-      '' + ''
-        export _NIX_TEST_UNIT_DATA=${./data}
-        ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
-        touch $out
-      '');
+      run =
+        runCommand "${finalAttrs.pname}-run"
+          {
+            meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages;
+          }
+          (
+            lib.optionalString stdenv.hostPlatform.isWindows ''
+              export HOME="$PWD/home-dir"
+              mkdir -p "$HOME"
+            ''
+            + ''
+              export _NIX_TEST_UNIT_DATA=${./data}
+              ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage}
+              touch $out
+            ''
+          );
     };
   };
 

src/libutil-tests/strings.cc

@@ -80,6 +80,42 @@ TEST(concatStringsSep, buildSingleString)
     ASSERT_EQ(concatStringsSep(",", strings), "this");
 }
 
+TEST(concatMapStringsSep, empty)
+{
+    Strings strings;
+
+    ASSERT_EQ(concatMapStringsSep(",", strings, [](const std::string & s) { return s; }), "");
+}
+
+TEST(concatMapStringsSep, justOne)
+{
+    Strings strings;
+    strings.push_back("this");
+
+    ASSERT_EQ(concatMapStringsSep(",", strings, [](const std::string & s) { return s; }), "this");
+}
+
+TEST(concatMapStringsSep, two)
+{
+    Strings strings;
+    strings.push_back("this");
+    strings.push_back("that");
+
+    ASSERT_EQ(concatMapStringsSep(",", strings, [](const std::string & s) { return s; }), "this,that");
+}
+
+TEST(concatMapStringsSep, map)
+{
+    std::map<std::string, std::string> strings;
+    strings["this"] = "that";
+    strings["1"] = "one";
+
+    ASSERT_EQ(
+        concatMapStringsSep(
+            ", ", strings, [](const std::pair<std::string, std::string> & s) { return s.first + " -> " + s.second; }),
+        "1 -> one, this -> that");
+}
+
 /* ----------------------------------------------------------------------------
  * dropEmptyInitThenConcatStringsSep
  * --------------------------------------------------------------------------*/

src/libutil/error.hh

@@ -50,6 +50,14 @@ struct LinesOfCode {
     std::optional<std::string> nextLineOfCode;
 };
 
+/* NOTE: position.hh recursively depends on source-path.hh -> source-accessor.hh
+   -> hash.hh -> config.hh -> experimental-features.hh -> error.hh -> Pos.
+   There are other such cycles.
+   Thus, Pos has to be an incomplete type in this header. But since ErrorInfo/Trace
+   have to refer to Pos, they have to use pointer indirection via std::shared_ptr
+   to break the recursive header dependency.
+   FIXME: Untangle this mess. Should there be AbstractPos as there used to be before
+   4feb7d9f71? */
 struct Pos;
 
 void printCodeLines(std::ostream & out,

src/libutil/file-content-address.cc

@@ -93,7 +93,7 @@ void restorePath(
 {
     switch (method) {
     case FileSerialisationMethod::Flat:
-        writeFile(path, source, 0666, startFsync);
+        writeFile(path, source, 0666, startFsync ? FsSync::Yes : FsSync::No);
         break;
     case FileSerialisationMethod::NixArchive:
         restorePath(path, source, startFsync);

src/libutil/file-system.cc

@@ -29,7 +29,17 @@
 
 namespace nix {
 
-namespace fs { using namespace std::filesystem; }
+namespace fs {
+  using namespace std::filesystem;
+
+  bool symlink_exists(const std::filesystem::path & path) {
+      try {
+          return std::filesystem::exists(std::filesystem::symlink_status(path));
+      } catch (const std::filesystem::filesystem_error & e) {
+          throw SysError("cannot check existence of %1%", path);
+      }
+  }
+}
 
 bool isAbsolute(PathView path)
 {
@@ -258,7 +268,7 @@ void readFile(const Path & path, Sink & sink)
 }
 
 
-void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
+void writeFile(const Path & path, std::string_view s, mode_t mode, FsSync sync)
 {
     AutoCloseFD fd = toDescriptor(open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT
 // TODO
@@ -268,22 +278,29 @@ void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
        , mode));
     if (!fd)
         throw SysError("opening file '%1%'", path);
-    try {
-        writeFull(fd.get(), s);
-    } catch (Error & e) {
-        e.addTrace({}, "writing file '%1%'", path);
-        throw;
-    }
-    if (sync)
-        fd.fsync();
-    // Explicitly close to make sure exceptions are propagated.
+
+    writeFile(fd, path, s, mode, sync);
+
+    /* Close explicitly to propagate the exceptions. */
     fd.close();
-    if (sync)
-        syncParent(path);
 }
 
+void writeFile(AutoCloseFD & fd, const Path & origPath, std::string_view s, mode_t mode, FsSync sync)
+{
+    assert(fd);
+    try {
+        writeFull(fd.get(), s);
 
-void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
+        if (sync == FsSync::Yes)
+            fd.fsync();
+
+    } catch (Error & e) {
+        e.addTrace({}, "writing file '%1%'", origPath);
+        throw;
+    }
+}
+
+void writeFile(const Path & path, Source & source, mode_t mode, FsSync sync)
 {
     AutoCloseFD fd = toDescriptor(open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT
 // TODO
@@ -307,11 +324,11 @@ void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
         e.addTrace({}, "writing file '%1%'", path);
         throw;
     }
-    if (sync)
+    if (sync == FsSync::Yes)
         fd.fsync();
     // Explicitly close to make sure exceptions are propagated.
     fd.close();
-    if (sync)
+    if (sync == FsSync::Yes)
         syncParent(path);
 }
 
@@ -374,7 +391,8 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b
 #ifndef _WIN32
     checkInterrupt();
 
-    std::string name(baseNameOf(path.native()));
+    std::string name(path.filename());
+    assert(name != "." && name != ".." && !name.empty());
 
     struct stat st;
     if (fstatat(parentfd, name.c_str(), &st,
@@ -415,7 +433,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b
                 throw SysError("chmod %1%", path);
         }
 
-        int fd = openat(parentfd, path.c_str(), O_RDONLY);
+        int fd = openat(parentfd, name.c_str(), O_RDONLY | O_DIRECTORY | O_NOFOLLOW);
         if (fd == -1)
             throw SysError("opening directory %1%", path);
         AutoCloseDir dir(fdopendir(fd));
@@ -427,7 +445,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b
             checkInterrupt();
             std::string childName = dirent->d_name;
             if (childName == "." || childName == "..") continue;
-            _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed);
+            _deletePath(dirfd(dir.get()), path / childName, bytesFreed);
         }
         if (errno) throw SysError("reading directory %1%", path);
     }
@@ -445,14 +463,13 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b
 
 static void _deletePath(const fs::path & path, uint64_t & bytesFreed)
 {
-    Path dir = dirOf(path.string());
-    if (dir == "")
-        dir = "/";
+    assert(path.is_absolute());
+    assert(path.parent_path() != path);
 
-    AutoCloseFD dirfd = toDescriptor(open(dir.c_str(), O_RDONLY));
+    AutoCloseFD dirfd = toDescriptor(open(path.parent_path().string().c_str(), O_RDONLY));
     if (!dirfd) {
         if (errno == ENOENT) return;
-        throw SysError("opening directory '%1%'", path);
+        throw SysError("opening directory %s", path.parent_path());
     }
 
     _deletePath(dirfd.get(), path, bytesFreed);
@@ -475,12 +492,12 @@ void createDir(const Path & path, mode_t mode)
         throw SysError("creating directory '%1%'", path);
 }
 
-void createDirs(const Path & path)
+void createDirs(const fs::path & path)
 {
     try {
         fs::create_directories(path);
     } catch (fs::filesystem_error & e) {
-        throw SysError("creating directory '%1%'", path);
+        throw SysError("creating directory '%1%'", path.string());
     }
 }
 
@@ -680,7 +697,7 @@ void setWriteTime(
         if (utimes(path.c_str(), times) == -1)
             throw SysError("changing modification time of %s (not a symlink)", path);
     } else {
-        throw Error("Cannot modification time of symlink %s", path);
+        throw Error("Cannot change modification time of symlink %s", path);
     }
 #endif
 #endif

src/libutil/file-system.hh

@@ -134,6 +134,7 @@ bool pathExists(const Path & path);
 namespace fs {
 
 /**
+ * TODO: we may actually want to use pathExists instead of this function
  *  ```
  *  symlink_exists(p) = std::filesystem::exists(std::filesystem::symlink_status(p))
  *  ```
@@ -142,9 +143,7 @@ namespace fs {
  *  std::filesystem::exists(p) = std::filesystem::exists(std::filesystem::status(p))
  *  ```
  */
-inline bool symlink_exists(const std::filesystem::path & path) {
-    return std::filesystem::exists(std::filesystem::symlink_status(path));
-}
+bool symlink_exists(const std::filesystem::path & path);
 
 } // namespace fs
 
@@ -194,21 +193,27 @@ std::string readFile(const Path & path);
 std::string readFile(const std::filesystem::path & path);
 void readFile(const Path & path, Sink & sink);
 
+enum struct FsSync { Yes, No };
+
 /**
  * Write a string to a file.
  */
-void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
-static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, bool sync = false)
+void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No);
+
+static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No)
 {
     return writeFile(path.string(), s, mode, sync);
 }
 
-void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
-static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, bool sync = false)
+void writeFile(const Path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No);
+
+static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, FsSync sync = FsSync::No)
 {
     return writeFile(path.string(), source, mode, sync);
 }
 
+void writeFile(AutoCloseFD & fd, const Path & origPath, std::string_view s, mode_t mode = 0666, FsSync sync = FsSync::No);
+
 /**
  * Flush a path's parent directory to disk.
  */
@@ -231,14 +236,9 @@ void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed);
 /**
  * Create a directory and all its parents, if necessary.
  *
- * In the process of being deprecated for
- * `std::filesystem::create_directories`.
+ * Wrapper around `std::filesystem::create_directories` to handle exceptions.
  */
-void createDirs(const Path & path);
-inline void createDirs(PathView path)
-{
-    return createDirs(Path(path));
-}
+void createDirs(const std::filesystem::path & path);
 
 /**
  * Create a single directory.

src/libutil/git.cc

@@ -33,7 +33,7 @@ std::optional<Mode> decodeMode(RawMode m) {
 static std::string getStringUntil(Source & source, char byte)
 {
     std::string s;
-    char n[1];
+    char n[1] = { 0 };
     source(std::string_view { n, 1 });
     while (*n != byte) {
         s += *n;

src/libutil/meson.build

@@ -148,6 +148,7 @@ sources = files(
   'logging.cc',
   'memory-source-accessor.cc',
   'position.cc',
+  'pos-table.cc',
   'posix-source-accessor.cc',
   'references.cc',
   'serialise.cc',
@@ -217,6 +218,8 @@ headers = [config_h] + files(
   'muxable-pipe.hh',
   'os-string.hh',
   'pool.hh',
+  'pos-idx.hh',
+  'pos-table.hh',
   'position.hh',
   'posix-source-accessor.hh',
   'processes.hh',

src/libutil/package.nix

@@ -1,18 +1,19 @@
-{ lib
-, stdenv
-, mkMesonLibrary
+{
+  lib,
+  stdenv,
+  mkMesonLibrary,
 
-, boost
-, brotli
-, libarchive
-, libcpuid
-, libsodium
-, nlohmann_json
-, openssl
+  boost,
+  brotli,
+  libarchive,
+  libcpuid,
+  libsodium,
+  nlohmann_json,
+  openssl,
 
-# Configuration Options
+  # Configuration Options
 
-, version
+  version,
 }:
 
 let
@@ -43,8 +44,7 @@ mkMesonLibrary (finalAttrs: {
     brotli
     libsodium
     openssl
-  ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
-  ;
+  ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid;
 
   propagatedBuildInputs = [
     boost
@@ -52,28 +52,10 @@ mkMesonLibrary (finalAttrs: {
     nlohmann_json
   ];
 
-  preConfigure =
-    # "Inline" .version so it's not a symlink, and includes the suffix.
-    # Do the meson utils, without modification.
-    #
-    # TODO: change release process to add `pre` in `.version`, remove it
-    # before tagging, and restore after.
-    ''
-      chmod u+w ./.version
-      echo ${version} > ../../.version
-    '';
-
   mesonFlags = [
     (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64)
   ];
 
-  env = {
-    # Needed for Meson to find Boost.
-    # https://github.com/NixOS/nixpkgs/issues/86131.
-    BOOST_INCLUDEDIR = "${lib.getDev boost}/include";
-    BOOST_LIBRARYDIR = "${lib.getLib boost}/lib";
-  };
-
   meta = {
     platforms = lib.platforms.unix ++ lib.platforms.windows;
   };