BuildResult: Add a "Cancelled" status

This denotes the result of a build that didn't succeed or fail, but
was cancelled because some other goal failed and --keep-going was not
enabled.

doc/manual/package.nix

@@ -58,8 +58,8 @@ mkMesonDerivation (finalAttrs: {
     "man"
   ];
 
-  # Hack for sake of the dev shell
-  passthru.externalNativeBuildInputs = [
+  nativeBuildInputs = [
+    nix-cli
     meson
     ninja
     (lib.getBin lowdown-unsandboxed)
@@ -78,10 +78,6 @@ mkMesonDerivation (finalAttrs: {
     changelog-d
   ];
 
-  nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [
-    nix-cli
-  ];
-
   preConfigure = ''
     chmod u+w ./.version
     echo ${finalAttrs.version} > ./.version

doc/manual/rl-next/json-format-changes.md

@@ -22,7 +22,15 @@ The store path info JSON format has been updated from version 1 to version 2:
   - New: `"ca": {"method": "nar", "hash": {"algorithm": "sha256", "format": "base64", "hash": "EMIJ+giQ..."}}`
   - Still `null` values for input-addressed store objects
 
-Version 1 format is still accepted when reading for backward compatibility.
+- **Structured hash fields**:
+
+  Hash values (`narHash` and `downloadHash`) are now structured JSON objects instead of strings:
+
+  - Old: `"narHash": "sha256:FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="`
+  - New: `"narHash": {"algorithm": "sha256", "format": "base64", "hash": "FePFYIlM..."}`
+  - Same structure applies to `downloadHash` in NAR info contexts
+
+Nix currently only produces, and doesn't consume this format.
 
 **Affected command**: `nix path-info --json`
 

doc/manual/rl-next/s3-storage-class.md

@@ -0,0 +1,21 @@
+---
+synopsis: "S3 binary cache stores now support storage class configuration"
+prs: [14464]
+issues: [7015]
+---
+
+S3 binary cache stores now support configuring the storage class for uploaded objects via the `storage-class` parameter. This allows users to optimize costs by selecting appropriate storage tiers based on access patterns.
+
+Example usage:
+
+```bash
+# Use Glacier storage for long-term archival
+nix copy --to 's3://my-bucket?storage-class=GLACIER' /nix/store/...
+
+# Use Intelligent Tiering for automatic cost optimization
+nix copy --to 's3://my-bucket?storage-class=INTELLIGENT_TIERING' /nix/store/...
+```
+
+The storage class applies to both regular uploads and multipart uploads. When not specified, objects use the bucket's default storage class.
+
+See the [S3 storage classes documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-class-intro.html) for available storage classes and their characteristics.

doc/manual/source/SUMMARY.md.in

@@ -29,6 +29,7 @@
   - [Build Trace](store/build-trace.md)
   - [Derivation Resolution](store/resolution.md)
   - [Building](store/building.md)
+  - [Secrets](store/secrets.md)
   - [Store Types](store/types/index.md)
 {{#include ./store/types/SUMMARY.md}}
   - [Appendix: Math notation](store/math-notation.md)

doc/manual/source/protocols/json/schema/store-object-info-v2.yaml

@@ -71,7 +71,7 @@ $defs:
           Note: This field may not be present in all contexts, such as when the path is used as the key and the the store object info the value in map.
 
       narHash:
-        type: string
+        "$ref": "./hash-v1.yaml"
         title: NAR Hash
         description: |
           Hash of the [file system object](@docroot@/store/file-system-object.md) part of the store object when serialized as a [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive).
@@ -229,7 +229,7 @@ $defs:
           > This is an impure "`.narinfo`" field that may not be included in certain contexts.
 
       downloadHash:
-        type: string
+        "$ref": "./hash-v1.yaml"
         title: Download Hash
         description: |
           A digest for the compressed archive itself, as opposed to the data contained within.

doc/manual/source/protocols/nix-archive/index.md

@@ -24,7 +24,7 @@ nar-obj-inner
   | str("type"), str("directory") directory
   ;
 
-regular = [ str("executable") ], str("contents"), str(contents);
+regular = [ str("executable"), str("") ], str("contents"), str(contents);
 
 symlink = str("target"), str(target);
 
@@ -52,4 +52,4 @@ The Nix Archive (NAR) format is also formally described using [Kaitai Struct](ht
 {{#include nar.ksy}}
 ```
 
-The source of the spec can be found [here](https://github.com/nixos/nix/blob/master/src/nix-manual/source/protocols/nix-archive/nar.ksy). Contributions and improvements to the spec are welcomed.
+The source of the spec can be found [here](https://github.com/nixos/nix/blob/master/src/nix-manual/source/protocols/nix-archive/nar.ksy). Contributions and improvements to the spec are welcomed.

doc/manual/source/store/secrets.md

@@ -0,0 +1,20 @@
+# Secrets
+
+The store is readable to all users on the system. For this reason, it
+is generally discouraged to allow secrets to make it into the store.
+
+Even on a single-user system, separate system users isolate services
+from each other and having secrets that all local users can read
+weakens that isolation. When using external store caches the secrets
+may end up there, and on multi-user systems the secrets will be
+available to all those users.
+
+Organize your derivations so that secrets are read from the filesystem
+(with appropriate access controls) at run time. Place the secrets on
+the filesystem manually or use a scheme that includes the secret in
+the store in encrypted form, and decrypts it adding the relevant
+access control on system activation.
+Several such schemes for NixOS can in the
+[comparison of secret managing schemes] on the wiki.
+
+[comparison of secret managing schemes]: https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes

meson.build

@@ -60,4 +60,9 @@ if get_option('unit-tests')
   subproject('libflake-tests')
 endif
 subproject('nix-functional-tests')
-subproject('json-schema-checks')
+if get_option('json-schema-checks')
+  subproject('json-schema-checks')
+endif
+if get_option('kaitai-struct-checks')
+  subproject('kaitai-struct-checks')
+endif

meson.options

@@ -27,3 +27,17 @@ option(
   value : false,
   description : 'Build benchmarks (requires gbenchmark)',
 )
+
+option(
+  'kaitai-struct-checks',
+  type : 'boolean',
+  value : true,
+  description : 'Check the Kaitai Struct specifications (requires Kaitai Struct)',
+)
+
+option(
+  'json-schema-checks',
+  type : 'boolean',
+  value : true,
+  description : 'Check JSON schema validity of schemas and examples (requires jv)',
+)

packaging/dev-shell.nix

@@ -3,10 +3,118 @@
   devFlake,
 }:
 
+let
+  # Some helper functions
+
+  /**
+    Compute a filtered closure of build inputs.
+
+    Specifically, `buildInputsClosure cond startSet` computes the closure formed
+    by recursive application of `p: filter cond p.buildInputs ++ filter cond p.propagatedBuildInputs`
+    to `startSet`.
+
+    Example:
+    ```nix
+    builtInputsClosure isInternal [ pkg1 pkg2 ]
+    => [ pkg1 pkg3 pkg2 pkg10 ]
+    ```
+
+    Note: order tbd
+
+    Note: `startSet` is *NOT* filtered.
+  */
+  buildInputsClosureCond =
+    cond: startSet:
+    let
+      closure = builtins.genericClosure {
+        startSet = map (d: {
+          key = d.drvPath;
+          value = d;
+        }) startSet;
+        operator =
+          d:
+          let
+            r =
+              map
+                (d': {
+                  key = d'.drvPath;
+                  value = d';
+                })
+                (
+                  lib.filter cond d.value.buildInputs or [ ] ++ lib.filter cond d.value.propagatedBuildInputs or [ ]
+                );
+          in
+          r;
+      };
+    in
+    map (item: item.value) closure;
+
+  /**
+    `[ pkg1 pkg2 ]` -> `{ "...-pkg2.drv" = null; "...-pkg1.drv" = null }`
+
+    Note: fairly arbitrary order (hash based). Use for efficient set membership test only.
+  */
+  byDrvPath =
+    l:
+    lib.listToAttrs (
+      map (c: {
+        name =
+          # Just a lookup key
+          builtins.unsafeDiscardStringContext c.drvPath;
+        value = null;
+      }) l
+    );
+
+  /**
+    Stable dedup.
+
+    Unlike `listToAttrs` -> `attrValues`, this preserves the input ordering,
+    which is more predictable ("deterministic") than e.g. sorting store paths,
+    whose hashes affect the ordering on every change.
+  */
+  # TODO: add to Nixpkgs lib, refer from uniqueStrings
+  dedupByString =
+    key: l:
+    let
+      r =
+        lib.foldl'
+          (
+            a@{ list, set }:
+            elem:
+            let
+              k = builtins.unsafeDiscardStringContext (key elem);
+            in
+            if set ? ${k} then
+              a
+            else
+              let
+                # Note: O(n²) copying. Use linkedLists to concat them in one go at the end.
+                # https://github.com/NixOS/nixpkgs/pull/452088
+                newList = [ elem ] ++ list;
+                newSet = set // {
+                  ${k} = null;
+                };
+              in
+              builtins.seq newList builtins.seq newSet {
+                list = newList;
+                set = newSet;
+              }
+          )
+          {
+            list = [ ];
+            set = { };
+          }
+          l;
+    in
+    r.list;
+
+in
+
 { pkgs }:
 
+# TODO: don't use nix-util for this?
 pkgs.nixComponents2.nix-util.overrideAttrs (
-  attrs:
+  finalAttrs: prevAttrs:
 
   let
     stdenv = pkgs.nixDependencies2.stdenv;
@@ -21,13 +129,89 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
       "-D${prefix}:${rest}";
     havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix;
     ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags;
+
+    activeComponents = buildInputsClosureCond isInternal (
+      lib.attrValues (finalAttrs.passthru.config.getComponents allComponents)
+    );
+
+    allComponents = lib.filterAttrs (k: v: lib.isDerivation v) pkgs.nixComponents2;
+    internalDrvs = byDrvPath (
+      # Drop the attr names (not present in buildInputs anyway)
+      lib.attrValues allComponents
+      ++ lib.concatMap (c: lib.attrValues c.tests or { }) (lib.attrValues allComponents)
+    );
+
+    isInternal =
+      dep: internalDrvs ? ${builtins.unsafeDiscardStringContext dep.drvPath or "_non-existent_"};
+
   in
   {
-    pname = "shell-for-" + attrs.pname;
+    pname = "shell-for-nix";
+
+    passthru = {
+      inherit activeComponents;
+
+      # We use this attribute to store non-derivation values like functions and
+      # perhaps other things that are primarily for overriding and not the shell.
+      config = {
+        # Default getComponents
+        getComponents =
+          c:
+          builtins.removeAttrs c (
+            lib.optionals (!havePerl) [ "nix-perl-bindings" ]
+            ++ lib.optionals (!buildCanExecuteHost) [ "nix-manual" ]
+          );
+      };
+
+      /**
+        Produce a devShell for a given set of nix components
+
+        Example:
+
+        ```nix
+        shell.withActiveComponents (c: {
+          inherit (c) nix-util;
+        })
+        ```
+      */
+      withActiveComponents =
+        f2:
+        finalAttrs.finalPackage.overrideAttrs (
+          finalAttrs: prevAttrs: {
+            passthru = prevAttrs.passthru // {
+              config = prevAttrs.passthru.config // {
+                getComponents = f2;
+              };
+            };
+          }
+        );
+
+      small =
+        (finalAttrs.finalPackage.withActiveComponents (c: {
+          inherit (c)
+            nix-cli
+            nix-util-tests
+            nix-store-tests
+            nix-expr-tests
+            nix-fetchers-tests
+            nix-flake-tests
+            nix-functional-tests
+            # Currently required
+            nix-perl-bindings
+            ;
+        })).overrideAttrs
+          (o: {
+            mesonFlags = o.mesonFlags ++ [
+              # TODO: infer from activeComponents or vice versa
+              "-Dkaitai-struct-checks=false"
+              "-Djson-schema-checks=false"
+            ];
+          });
+    };
 
     # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop
     version = lib.fileContents ../.version;
-    name = attrs.pname;
+    name = finalAttrs.pname;
 
     installFlags = "sysconfdir=$(out)/etc";
     shellHook = ''
@@ -98,17 +282,9 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
     nativeBuildInputs =
       let
         inputs =
-          attrs.nativeBuildInputs or [ ]
-          ++ pkgs.nixComponents2.nix-util.nativeBuildInputs
-          ++ pkgs.nixComponents2.nix-store.nativeBuildInputs
-          ++ pkgs.nixComponents2.nix-fetchers.nativeBuildInputs
-          ++ pkgs.nixComponents2.nix-expr.nativeBuildInputs
-          ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.nativeBuildInputs
-          ++ lib.optionals buildCanExecuteHost pkgs.nixComponents2.nix-manual.externalNativeBuildInputs
-          ++ pkgs.nixComponents2.nix-internal-api-docs.nativeBuildInputs
-          ++ pkgs.nixComponents2.nix-external-api-docs.nativeBuildInputs
-          ++ pkgs.nixComponents2.nix-functional-tests.externalNativeBuildInputs
-          ++ pkgs.nixComponents2.nix-json-schema-checks.externalNativeBuildInputs
+          dedupByString (v: "${v}") (
+            lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.nativeBuildInputs) activeComponents)
+          )
           ++ lib.optional (
             !buildCanExecuteHost
             # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479
@@ -117,9 +293,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
             && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)
           ) pkgs.buildPackages.mesonEmulatorHook
           ++ [
-            pkgs.buildPackages.cmake
             pkgs.buildPackages.gnused
-            pkgs.buildPackages.changelog-d
             modular.pre-commit.settings.package
             (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript)
             pkgs.buildPackages.nixfmt-rfc-style
@@ -136,18 +310,22 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
       # from making its way into NIX_CFLAGS_COMPILE.
       lib.filter (p: !lib.hasInfix "separate-debug-info" p) inputs;
 
+    propagatedNativeBuildInputs = dedupByString (v: "${v}") (
+      lib.filter (x: !isInternal x) (
+        lib.lists.concatMap (c: c.propagatedNativeBuildInputs) activeComponents
+      )
+    );
+
     buildInputs = [
       pkgs.gbenchmark
     ]
-    ++ attrs.buildInputs or [ ]
-    ++ pkgs.nixComponents2.nix-util.buildInputs
-    ++ pkgs.nixComponents2.nix-store.buildInputs
-    ++ pkgs.nixComponents2.nix-store-tests.externalBuildInputs
-    ++ pkgs.nixComponents2.nix-fetchers.buildInputs
-    ++ pkgs.nixComponents2.nix-expr.buildInputs
-    ++ pkgs.nixComponents2.nix-expr.externalPropagatedBuildInputs
-    ++ pkgs.nixComponents2.nix-cmd.buildInputs
-    ++ lib.optionals havePerl pkgs.nixComponents2.nix-perl-bindings.externalBuildInputs
+    ++ dedupByString (v: "${v}") (
+      lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.buildInputs) activeComponents)
+    )
     ++ lib.optional havePerl pkgs.perl;
+
+    propagatedBuildInputs = dedupByString (v: "${v}") (
+      lib.filter (x: !isInternal x) (lib.lists.concatMap (c: c.propagatedBuildInputs) activeComponents)
+    );
   }
 )

src/json-schema-checks/package.nix

@@ -34,15 +34,11 @@ mkMesonDerivation (finalAttrs: {
 
   outputs = [ "out" ];
 
-  passthru.externalNativeBuildInputs = [
-    jsonschema
-  ];
-
   nativeBuildInputs = [
     meson
     ninja
-  ]
-  ++ finalAttrs.passthru.externalNativeBuildInputs;
+    jsonschema
+  ];
 
   doCheck = true;
 

src/kaitai-struct-checks/package.nix

@@ -37,7 +37,15 @@ mkMesonDerivation (finalAttrs: {
 
   outputs = [ "out" ];
 
-  passthru.externalNativeBuildInputs = [
+  buildInputs = [
+    gtest
+    kaitai-struct-cpp-stl-runtime
+  ];
+
+  nativeBuildInputs = [
+    meson
+    ninja
+    pkg-config
     # This can go away when we bump up to 25.11
     (kaitai-struct-compiler.overrideAttrs (finalAttrs: {
       version = "0.11";
@@ -48,20 +56,6 @@ mkMesonDerivation (finalAttrs: {
     }))
   ];
 
-  passthru.externalBuildInputs = [
-    gtest
-    kaitai-struct-cpp-stl-runtime
-  ];
-
-  buildInputs = finalAttrs.passthru.externalBuildInputs;
-
-  nativeBuildInputs = [
-    meson
-    ninja
-    pkg-config
-  ]
-  ++ finalAttrs.passthru.externalNativeBuildInputs;
-
   doCheck = true;
 
   mesonCheckFlags = [ "--print-errorlogs" ];

src/libcmd/command.cc

@@ -2,6 +2,7 @@
 #include <nlohmann/json.hpp>
 
 #include "nix/cmd/command.hh"
+#include "nix/cmd/legacy.hh"
 #include "nix/cmd/markdown.hh"
 #include "nix/store/store-open.hh"
 #include "nix/store/local-fs-store.hh"
@@ -14,6 +15,18 @@
 
 namespace nix {
 
+RegisterCommand::Commands & RegisterCommand::commands()
+{
+    static RegisterCommand::Commands commands;
+    return commands;
+}
+
+RegisterLegacyCommand::Commands & RegisterLegacyCommand::commands()
+{
+    static RegisterLegacyCommand::Commands commands;
+    return commands;
+}
+
 nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
 {
     nix::Commands res;

src/libcmd/include/nix/cmd/command.hh

@@ -286,11 +286,7 @@ struct RegisterCommand
 {
     typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
 
-    static Commands & commands()
-    {
-        static Commands commands;
-        return commands;
-    }
+    static Commands & commands();
 
     RegisterCommand(std::vector<std::string> && name, std::function<ref<Command>()> command)
     {

src/libcmd/include/nix/cmd/legacy.hh

@@ -13,11 +13,7 @@ struct RegisterLegacyCommand
 {
     typedef std::map<std::string, MainFunction> Commands;
 
-    static Commands & commands()
-    {
-        static Commands commands;
-        return commands;
-    }
+    static Commands & commands();
 
     RegisterLegacyCommand(const std::string & name, MainFunction fun)
     {

src/libexpr-test-support/include/nix/expr/tests/libexpr.hh

@@ -104,6 +104,7 @@ MATCHER(IsAttrs, "")
 MATCHER_P(IsStringEq, s, fmt("The string is equal to \"%1%\"", s))
 {
     if (arg.type() != nString) {
+        *result_listener << "Expected a string got " << arg.type();
         return false;
     }
     return arg.string_view() == s;

src/libexpr-tests/json.cc

@@ -1,5 +1,6 @@
 #include "nix/expr/tests/libexpr.hh"
 #include "nix/expr/value-to-json.hh"
+#include "nix/expr/static-string-data.hh"
 
 namespace nix {
 // Testing the conversion to JSON
@@ -54,7 +55,7 @@ TEST_F(JSONValueTest, IntNegative)
 TEST_F(JSONValueTest, String)
 {
     Value v;
-    v.mkStringNoCopy("test");
+    v.mkStringNoCopy("test"_sds);
     ASSERT_EQ(getJSONValue(v), "\"test\"");
 }
 
@@ -62,7 +63,7 @@ TEST_F(JSONValueTest, StringQuotes)
 {
     Value v;
 
-    v.mkStringNoCopy("test\"");
+    v.mkStringNoCopy("test\""_sds);
     ASSERT_EQ(getJSONValue(v), "\"test\\\"\"");
 }
 

src/libexpr-tests/value/print.cc

@@ -1,4 +1,5 @@
 #include "nix/expr/tests/libexpr.hh"
+#include "nix/expr/static-string-data.hh"
 
 #include "nix/expr/value.hh"
 #include "nix/expr/print.hh"
@@ -35,14 +36,14 @@ TEST_F(ValuePrintingTests, tBool)
 TEST_F(ValuePrintingTests, tString)
 {
     Value vString;
-    vString.mkStringNoCopy("some-string");
+    vString.mkStringNoCopy("some-string"_sds);
     test(vString, "\"some-string\"");
 }
 
 TEST_F(ValuePrintingTests, tPath)
 {
     Value vPath;
-    vPath.mkStringNoCopy("/foo");
+    vPath.mkStringNoCopy("/foo"_sds);
     test(vPath, "\"/foo\"");
 }
 
@@ -289,10 +290,10 @@ TEST_F(StringPrintingTests, maxLengthTruncation)
 TEST_F(ValuePrintingTests, attrsTypeFirst)
 {
     Value vType;
-    vType.mkStringNoCopy("puppy");
+    vType.mkStringNoCopy("puppy"_sds);
 
     Value vApple;
-    vApple.mkStringNoCopy("apple");
+    vApple.mkStringNoCopy("apple"_sds);
 
     BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.symbols.create("type"), &vType);
@@ -333,7 +334,7 @@ TEST_F(ValuePrintingTests, ansiColorsBool)
 TEST_F(ValuePrintingTests, ansiColorsString)
 {
     Value v;
-    v.mkStringNoCopy("puppy");
+    v.mkStringNoCopy("puppy"_sds);
 
     test(v, ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, PrintOptions{.ansiColors = true});
 }
@@ -341,7 +342,7 @@ TEST_F(ValuePrintingTests, ansiColorsString)
 TEST_F(ValuePrintingTests, ansiColorsStringElided)
 {
     Value v;
-    v.mkStringNoCopy("puppy");
+    v.mkStringNoCopy("puppy"_sds);
 
     test(
         v,
@@ -389,7 +390,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs)
 TEST_F(ValuePrintingTests, ansiColorsDerivation)
 {
     Value vDerivation;
-    vDerivation.mkStringNoCopy("derivation");
+    vDerivation.mkStringNoCopy("derivation"_sds);
 
     BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.s.type, &vDerivation);
@@ -412,7 +413,7 @@ TEST_F(ValuePrintingTests, ansiColorsError)
 {
     Value throw_ = state.getBuiltin("throw");
     Value message;
-    message.mkStringNoCopy("uh oh!");
+    message.mkStringNoCopy("uh oh!"_sds);
     Value vError;
     vError.mkApp(&throw_, &message);
 
@@ -429,12 +430,12 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError)
 {
     Value throw_ = state.getBuiltin("throw");
     Value message;
-    message.mkStringNoCopy("uh oh!");
+    message.mkStringNoCopy("uh oh!"_sds);
     Value vError;
     vError.mkApp(&throw_, &message);
 
     Value vDerivation;
-    vDerivation.mkStringNoCopy("derivation");
+    vDerivation.mkStringNoCopy("derivation"_sds);
 
     BindingsBuilder builder = state.buildBindings(10);
     builder.insert(state.s.type, &vDerivation);

src/libexpr-tests/value/value.cc

@@ -1,4 +1,5 @@
 #include "nix/expr/value.hh"
+#include "nix/expr/static-string-data.hh"
 
 #include "nix/store/tests/libstore.hh"
 #include <gtest/gtest.h>
@@ -27,17 +28,17 @@ TEST_F(ValueTest, staticString)
 {
     Value vStr1;
     Value vStr2;
-    vStr1.mkStringNoCopy("foo");
-    vStr2.mkStringNoCopy("foo");
+    vStr1.mkStringNoCopy("foo"_sds);
+    vStr2.mkStringNoCopy("foo"_sds);
 
-    auto sd1 = vStr1.string_view();
-    auto sd2 = vStr2.string_view();
+    auto & sd1 = vStr1.string_data();
+    auto & sd2 = vStr2.string_data();
 
     // The strings should be the same
-    ASSERT_EQ(sd1, sd2);
+    ASSERT_EQ(sd1.view(), sd2.view());
 
     // The strings should also be backed by the same (static) allocation
-    ASSERT_EQ(sd1.data(), sd2.data());
+    ASSERT_EQ(&sd1, &sd2);
 }
 
 } // namespace nix

src/libexpr/eval-cache.cc

@@ -147,7 +147,7 @@ struct AttrDb
                 for (auto * elem : *context) {
                     if (!first)
                         ctx.push_back(' ');
-                    ctx.append(elem);
+                    ctx.append(elem->view());
                     first = false;
                 }
                 state->insertAttributeWithContext.use()(key.first)(symbols[key.second])(AttrType::String) (s) (ctx)

src/libexpr/eval.cc

@@ -3,6 +3,7 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/print-options.hh"
 #include "nix/expr/symbol-table.hh"
+#include "nix/expr/value.hh"
 #include "nix/util/exit.hh"
 #include "nix/util/types.hh"
 #include "nix/util/util.hh"
@@ -28,6 +29,8 @@
 #include "parser-tab.hh"
 
 #include <algorithm>
+#include <cstddef>
+#include <cstdlib>
 #include <iostream>
 #include <sstream>
 #include <cstring>
@@ -48,6 +51,9 @@ using json = nlohmann::json;
 
 namespace nix {
 
+/**
+ * Just for doc strings. Not for regular string values.
+ */
 static char * allocString(size_t size)
 {
     char * t;
@@ -61,6 +67,9 @@ static char * allocString(size_t size)
 // string allocations.
 // This function handles makeImmutableString(std::string_view()) by returning
 // the empty string.
+/**
+ * Just for doc strings. Not for regular string values.
+ */
 static const char * makeImmutableString(std::string_view s)
 {
     const size_t size = s.size();
@@ -72,6 +81,25 @@ static const char * makeImmutableString(std::string_view s)
     return t;
 }
 
+StringData & StringData::alloc(size_t size)
+{
+    void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
+    if (!t)
+        throw std::bad_alloc();
+    auto res = new (t) StringData(size);
+    return *res;
+}
+
+const StringData & StringData::make(std::string_view s)
+{
+    if (s.empty())
+        return ""_sds;
+    auto & res = alloc(s.size());
+    std::memcpy(&res.data_, s.data(), s.size());
+    res.data_[s.size()] = '\0';
+    return res;
+}
+
 RootValue allocRootValue(Value * v)
 {
     return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v);
@@ -585,7 +613,9 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             .name = name,
             .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
             .args = {},
-            .doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC
+            /* N.B. Can't use StringData here, because that would lead to an interior pointer.
+               NOTE: memory leak when compiled without GC. */
+            .doc = makeImmutableString(s.view()),
         };
     }
     if (isFunctor(v)) {
@@ -819,33 +849,34 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
 
 void Value::mkString(std::string_view s)
 {
-    mkStringNoCopy(makeImmutableString(s));
+    mkStringNoCopy(StringData::make(s));
 }
 
-Value::StringWithContext::Context * Value::StringWithContext::Context::fromBuilder(const NixStringContext & context)
+Value::StringWithContext::Context *
+Value::StringWithContext::Context::fromBuilder(const NixStringContext & context, EvalMemory & mem)
 {
     if (context.empty())
         return nullptr;
 
-    auto ctx = new (allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
+    auto ctx = new (mem.allocBytes(sizeof(Context) + context.size() * sizeof(value_type))) Context(context.size());
     std::ranges::transform(
-        context, ctx->elems, [](const NixStringContextElem & elt) { return makeImmutableString(elt.to_string()); });
+        context, ctx->elems, [](const NixStringContextElem & elt) { return &StringData::make(elt.to_string()); });
     return ctx;
 }
 
-void Value::mkString(std::string_view s, const NixStringContext & context)
+void Value::mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem)
 {
-    mkStringNoCopy(makeImmutableString(s), Value::StringWithContext::Context::fromBuilder(context));
+    mkStringNoCopy(StringData::make(s), Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
-void Value::mkStringMove(const char * s, const NixStringContext & context)
+void Value::mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem)
 {
-    mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context));
+    mkStringNoCopy(s, Value::StringWithContext::Context::fromBuilder(context, mem));
 }
 
 void Value::mkPath(const SourcePath & path)
 {
-    mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
+    mkPath(&*path.accessor, StringData::make(path.path.abs()));
 }
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
@@ -881,9 +912,9 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
     }
 }
 
-ListBuilder::ListBuilder(size_t size)
+ListBuilder::ListBuilder(EvalMemory & mem, size_t size)
     : size(size)
-    , elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *)))
+    , elems(size <= 2 ? inlineElems : (Value **) mem.allocBytes(size * sizeof(Value *)))
 {
 }
 
@@ -923,7 +954,8 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v)
         store->printStorePath(p),
         NixStringContext{
             NixStringContextElem::Opaque{.path = p},
-        });
+        },
+        mem);
 }
 
 std::string EvalState::mkOutputStringRaw(
@@ -945,7 +977,7 @@ void EvalState::mkOutputString(
     std::optional<StorePath> optStaticOutputPath,
     const ExperimentalFeatureSettings & xpSettings)
 {
-    value.mkString(mkOutputStringRaw(b, optStaticOutputPath, xpSettings), NixStringContext{b});
+    value.mkString(mkOutputStringRaw(b, optStaticOutputPath, xpSettings), NixStringContext{b}, mem);
 }
 
 std::string EvalState::mkSingleDerivedPathStringRaw(const SingleDerivedPath & p)
@@ -980,7 +1012,8 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v
         mkSingleDerivedPathStringRaw(p),
         NixStringContext{
             std::visit([](auto && v) -> NixStringContextElem { return v; }, p),
-        });
+        },
+        mem);
 }
 
 Value * Expr::maybeThunk(EvalState & state, Env & env)
@@ -2099,21 +2132,21 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 .atPos(pos)
                 .withFrame(env, *this)
                 .debugThrow();
-        std::string result_str;
-        result_str.reserve(sSize);
+        std::string resultStr;
+        resultStr.reserve(sSize);
         for (const auto & part : strings) {
-            result_str += *part;
+            resultStr += *part;
         }
-        v.mkPath(state.rootPath(CanonPath(result_str)));
+        v.mkPath(state.rootPath(CanonPath(resultStr)));
     } else {
-        char * result_str = allocString(sSize + 1);
-        char * tmp = result_str;
+        auto & resultStr = StringData::alloc(sSize);
+        auto * tmp = resultStr.data();
         for (const auto & part : strings) {
-            memcpy(tmp, part->data(), part->size());
+            std::memcpy(tmp, part->data(), part->size());
             tmp += part->size();
         }
-        *tmp = 0;
-        v.mkStringMove(result_str, context);
+        *tmp = '\0';
+        v.mkStringMove(resultStr, context, state.mem);
     }
 }
 
@@ -2288,7 +2321,7 @@ void copyContext(const Value & v, NixStringContext & context, const Experimental
 {
     if (auto * ctx = v.context())
         for (auto * elem : *ctx)
-            context.insert(NixStringContextElem::parse(elem, xpSettings));
+            context.insert(NixStringContextElem::parse(elem->view(), xpSettings));
 }
 
 std::string_view EvalState::forceString(
@@ -2310,7 +2343,7 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s
         error<EvalError>(
             "the string '%1%' is not allowed to refer to a store path (such as '%2%')",
             v.string_view(),
-            *v.context()->begin())
+            (*v.context()->begin())->view())
             .withTrace(pos, errorCtx)
             .debugThrow();
     }

src/libexpr/include/nix/expr/eval-inline.hh

@@ -12,7 +12,7 @@ namespace nix {
  * Note: Various places expect the allocated memory to be zeroed.
  */
 [[gnu::always_inline]]
-inline void * allocBytes(size_t n)
+inline void * EvalMemory::allocBytes(size_t n)
 {
     void * p;
 #if NIX_USE_BOEHMGC

src/libexpr/include/nix/expr/eval.hh

@@ -335,6 +335,7 @@ public:
     EvalMemory & operator=(const EvalMemory &) = delete;
     EvalMemory & operator=(EvalMemory &&) = delete;
 
+    inline void * allocBytes(size_t n);
     inline Value * allocValue();
     inline Env & allocEnv(size_t size);
 
@@ -348,7 +349,7 @@ public:
     ListBuilder buildList(size_t size)
     {
         stats.nrListElems += size;
-        return ListBuilder(size);
+        return ListBuilder(*this, size);
     }
 
     const Statistics & getStats() const &

src/libexpr/include/nix/expr/meson.build

@@ -31,6 +31,7 @@ headers = [ config_pub_h ] + files(
   'print.hh',
   'repl-exit-status.hh',
   'search-path.hh',
+  'static-string-data.hh',
   'symbol-table.hh',
   'value-to-json.hh',
   'value-to-xml.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -3,6 +3,7 @@
 
 #include <map>
 #include <span>
+#include <memory>
 #include <vector>
 #include <memory_resource>
 #include <algorithm>
@@ -11,6 +12,7 @@
 #include "nix/expr/value.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
+#include "nix/expr/static-string-data.hh"
 #include "nix/util/pos-idx.hh"
 #include "nix/expr/counter.hh"
 #include "nix/util/pos-table.hh"
@@ -186,22 +188,18 @@ struct ExprString : Expr
      * This is only for strings already allocated in our polymorphic allocator,
      * or that live at least that long (e.g. c++ string literals)
      */
-    ExprString(const char * s)
+    ExprString(const StringData & s)
     {
         v.mkStringNoCopy(s);
     };
 
     ExprString(std::pmr::polymorphic_allocator<char> & alloc, std::string_view sv)
     {
-        auto len = sv.length();
-        if (len == 0) {
-            v.mkStringNoCopy("");
+        if (sv.size() == 0) {
+            v.mkStringNoCopy(""_sds);
             return;
         }
-        char * s = alloc.allocate(len + 1);
-        sv.copy(s, len);
-        s[len] = '\0';
-        v.mkStringNoCopy(s);
+        v.mkStringNoCopy(StringData::make(*alloc.resource(), sv));
     };
 
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -216,11 +214,7 @@ struct ExprPath : Expr
     ExprPath(std::pmr::polymorphic_allocator<char> & alloc, ref<SourceAccessor> accessor, std::string_view sv)
         : accessor(accessor)
     {
-        auto len = sv.length();
-        char * s = alloc.allocate(len + 1);
-        sv.copy(s, len);
-        s[len] = '\0';
-        v.mkPath(&*accessor, s);
+        v.mkPath(&*accessor, StringData::make(*alloc.resource(), sv));
     }
 
     Value * maybeThunk(EvalState & state, Env & env) override;
@@ -345,7 +339,7 @@ struct ExprOpHasAttr : Expr
     Expr * e;
     std::span<AttrName> attrPath;
 
-    ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::vector<AttrName> attrPath)
+    ExprOpHasAttr(std::pmr::polymorphic_allocator<char> & alloc, Expr * e, std::span<AttrName> attrPath)
         : e(e)
         , attrPath({alloc.allocate_object<AttrName>(attrPath.size()), attrPath.size()})
     {
@@ -439,7 +433,7 @@ struct ExprList : Expr
 {
     std::span<Expr *> elems;
 
-    ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::vector<Expr *> exprs)
+    ExprList(std::pmr::polymorphic_allocator<char> & alloc, std::span<Expr *> exprs)
         : elems({alloc.allocate_object<Expr *>(exprs.size()), exprs.size()})
     {
         std::ranges::copy(exprs, elems.begin());
@@ -568,7 +562,7 @@ public:
         const PosTable & positions,
         std::pmr::polymorphic_allocator<char> & alloc,
         PosIdx pos,
-        FormalsBuilder formals,
+        const FormalsBuilder & formals,
         Expr * body)
         : ExprLambda(positions, alloc, pos, Symbol(), formals, body) {};
 
@@ -759,7 +753,19 @@ struct ExprConcatStrings : Expr
         std::pmr::polymorphic_allocator<char> & alloc,
         const PosIdx & pos,
         bool forceString,
-        const std::vector<std::pair<PosIdx, Expr *>> & es)
+        std::span<std::pair<PosIdx, Expr *>> es)
+        : pos(pos)
+        , forceString(forceString)
+        , es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()})
+    {
+        std::ranges::copy(es, this->es.begin());
+    };
+
+    ExprConcatStrings(
+        std::pmr::polymorphic_allocator<char> & alloc,
+        const PosIdx & pos,
+        bool forceString,
+        std::initializer_list<std::pair<PosIdx, Expr *>> es)
         : pos(pos)
         , forceString(forceString)
         , es({alloc.allocate_object<std::pair<PosIdx, Expr *>>(es.size()), es.size()})
@@ -839,7 +845,19 @@ public:
     add(std::pmr::polymorphic_allocator<char> & alloc,
         const PosIdx & pos,
         bool forceString,
-        const std::vector<std::pair<PosIdx, Expr *>> & es)
+        std::span<std::pair<PosIdx, Expr *>> es)
+        requires(std::same_as<C, ExprConcatStrings>)
+    {
+        return alloc.new_object<C>(alloc, pos, forceString, es);
+    }
+
+    template<class C>
+    [[gnu::always_inline]]
+    C *
+    add(std::pmr::polymorphic_allocator<char> & alloc,
+        const PosIdx & pos,
+        bool forceString,
+        std::initializer_list<std::pair<PosIdx, Expr *>> es)
         requires(std::same_as<C, ExprConcatStrings>)
     {
         return alloc.new_object<C>(alloc, pos, forceString, es);

src/libexpr/include/nix/expr/parser-state.hh

@@ -4,6 +4,8 @@
 #include <limits>
 
 #include "nix/expr/eval.hh"
+#include "nix/expr/value.hh"
+#include "nix/expr/static-string-data.hh"
 
 namespace nix {
 
@@ -94,7 +96,7 @@ struct ParserState
         ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc);
     void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def);
     void validateFormals(FormalsBuilder & formals, PosIdx pos = noPos, Symbol arg = {});
-    Expr * stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es);
+    Expr * stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std::variant<Expr *, StringToken>>> es);
     PosIdx at(const ParserLocation & loc);
 };
 
@@ -237,10 +239,10 @@ inline void ParserState::validateFormals(FormalsBuilder & formals, PosIdx pos, S
 }
 
 inline Expr *
-ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es)
+ParserState::stripIndentation(const PosIdx pos, std::span<std::pair<PosIdx, std::variant<Expr *, StringToken>>> es)
 {
     if (es.empty())
-        return exprs.add<ExprString>("");
+        return exprs.add<ExprString>(""_sds);
 
     /* Figure out the minimum indentation.  Note that by design
        whitespace-only final lines are not taken into account.  (So
@@ -332,7 +334,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
     // If there is nothing at all, return the empty string directly.
     // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters.
     if (es2.size() == 0) {
-        auto * const result = exprs.add<ExprString>("");
+        auto * const result = exprs.add<ExprString>(""_sds);
         return result;
     }
 
@@ -341,7 +343,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vector<std::pair<PosIdx, st
         auto * const result = (es2)[0].second;
         return result;
     }
-    return exprs.add<ExprConcatStrings>(exprs.alloc, pos, true, std::move(es2));
+    return exprs.add<ExprConcatStrings>(exprs.alloc, pos, true, es2);
 }
 
 inline PosIdx LexerState::at(const ParserLocation & loc)

src/libexpr/include/nix/expr/primops.hh

@@ -12,11 +12,7 @@ struct RegisterPrimOp
 {
     typedef std::vector<PrimOp> PrimOps;
 
-    static PrimOps & primOps()
-    {
-        static PrimOps primOps;
-        return primOps;
-    }
+    static PrimOps & primOps();
 
     /**
      * You can register a constant by passing an arity of 0. fun

src/libexpr/include/nix/expr/print-options.hh

@@ -110,7 +110,7 @@ struct PrintOptions
  * `PrintOptions` for unknown and therefore potentially large values in error messages,
  * to avoid printing "too much" output.
  */
-static PrintOptions errorPrintOptions = PrintOptions{
+static constexpr PrintOptions errorPrintOptions = PrintOptions{
     .ansiColors = true,
     .maxDepth = 10,
     .maxAttrs = 10,

src/libexpr/include/nix/expr/static-string-data.hh

@@ -0,0 +1,44 @@
+#pragma once
+///@file
+
+#include "nix/expr/value.hh"
+
+namespace nix {
+
+template<size_t N>
+struct StringData::Static
+{
+    /**
+     * @note Must be first to make layout compatible with StringData.
+     */
+    const size_t size = N - 1;
+    char data[N];
+
+    consteval Static(const char (&str)[N])
+    {
+        static_assert(N > 0);
+        if (str[size] != '\0')
+            throw;
+        std::copy_n(str, N, data);
+    }
+
+    operator const StringData &() const &
+    {
+        static_assert(sizeof(decltype(*this)) >= sizeof(StringData));
+        static_assert(alignof(decltype(*this)) == alignof(StringData));
+        /* NOTE: This cast is somewhat on the fence of what's legal in C++.
+           The question boils down to whether flexible array members are
+           layout compatible with fixed-size arrays. This is a gray area, since
+           FAMs are not standard anyway.
+           */
+        return *reinterpret_cast<const StringData *>(this);
+    }
+};
+
+template<StringData::Static S>
+const StringData & operator""_sds()
+{
+    return S;
+}
+
+} // namespace nix

src/libexpr/include/nix/expr/symbol-table.hh

@@ -3,6 +3,7 @@
 
 #include <memory_resource>
 #include "nix/expr/value.hh"
+#include "nix/expr/static-string-data.hh"
 #include "nix/util/chunked-vector.hh"
 #include "nix/util/error.hh"
 
@@ -16,7 +17,6 @@ class SymbolValue : protected Value
     friend class SymbolStr;
     friend class SymbolTable;
 
-    uint32_t size_;
     uint32_t idx;
 
     SymbolValue() = default;
@@ -24,7 +24,7 @@ class SymbolValue : protected Value
 public:
     operator std::string_view() const noexcept
     {
-        return {c_str(), size_};
+        return string_view();
     }
 };
 
@@ -96,13 +96,13 @@ class SymbolStr
         SymbolValueStore & store;
         std::string_view s;
         std::size_t hash;
-        std::pmr::polymorphic_allocator<char> & alloc;
+        std::pmr::memory_resource & resource;
 
-        Key(SymbolValueStore & store, std::string_view s, std::pmr::polymorphic_allocator<char> & stringAlloc)
+        Key(SymbolValueStore & store, std::string_view s, std::pmr::memory_resource & stringMemory)
             : store(store)
             , s(s)
             , hash(HashType{}(s))
-            , alloc(stringAlloc)
+            , resource(stringMemory)
         {
         }
     };
@@ -122,14 +122,10 @@ public:
         // for multi-threaded implementations: lock store and allocator here
         const auto & [v, idx] = key.store.add(SymbolValue{});
         if (size == 0) {
-            v.mkStringNoCopy("", nullptr);
+            v.mkStringNoCopy(""_sds, nullptr);
         } else {
-            auto s = key.alloc.allocate(size + 1);
-            memcpy(s, key.s.data(), size);
-            s[size] = '\0';
-            v.mkStringNoCopy(s, nullptr);
+            v.mkStringNoCopy(StringData::make(key.resource, key.s));
         }
-        v.size_ = size;
         v.idx = idx;
         this->s = &v;
     }
@@ -139,6 +135,12 @@ public:
         return *s == s2;
     }
 
+    [[gnu::always_inline]]
+    const StringData & string_data() const noexcept
+    {
+        return s->string_data();
+    }
+
     [[gnu::always_inline]]
     const char * c_str() const noexcept
     {
@@ -155,13 +157,17 @@ public:
     [[gnu::always_inline]]
     bool empty() const noexcept
     {
-        return s->size_ == 0;
+        auto * p = &s->string_data();
+        // Save a dereference in the sentinel value case
+        if (p == &""_sds)
+            return true;
+        return p->size() == 0;
     }
 
     [[gnu::always_inline]]
     size_t size() const noexcept
     {
-        return s->size_;
+        return s->string_data().size();
     }
 
     [[gnu::always_inline]]
@@ -259,7 +265,6 @@ private:
      * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based.
      */
     std::pmr::monotonic_buffer_resource buffer;
-    std::pmr::polymorphic_allocator<char> stringAlloc{&buffer};
     SymbolStr::SymbolValueStore store{16};
 
     /**
@@ -282,7 +287,7 @@ public:
         // Most symbols are looked up more than once, so we trade off insertion performance
         // for lookup performance.
         // FIXME: make this thread-safe.
-        return Symbol(*symbols.insert(SymbolStr::Key{store, s, stringAlloc}).first);
+        return Symbol(*symbols.insert(SymbolStr::Key{store, s, buffer}).first);
     }
 
     std::vector<SymbolStr> resolve(const std::vector<Symbol> & symbols) const

src/libexpr/include/nix/expr/value.hh

@@ -1,8 +1,14 @@
 #pragma once
 ///@file
 
+#include <bit>
 #include <cassert>
+#include <cstddef>
+#include <cstring>
+#include <memory>
+#include <memory_resource>
 #include <span>
+#include <string_view>
 #include <type_traits>
 #include <concepts>
 
@@ -82,6 +88,7 @@ class PosIdx;
 struct Pos;
 class StorePath;
 class EvalState;
+class EvalMemory;
 class XMLWriter;
 class Printer;
 
@@ -155,7 +162,7 @@ class ListBuilder
     Value * inlineElems[2] = {nullptr, nullptr};
 public:
     Value ** elems;
-    ListBuilder(size_t size);
+    ListBuilder(EvalMemory & mem, size_t size);
 
     // NOTE: Can be noexcept because we are just copying integral values and
     // raw pointers.
@@ -186,6 +193,91 @@ public:
     friend struct Value;
 };
 
+class StringData
+{
+public:
+    using size_type = std::size_t;
+
+    size_type size_;
+    char data_[];
+
+    /*
+     * This in particular ensures that we cannot have a `StringData`
+     * that we use by value, which is just what we want!
+     *
+     * Dynamically sized types aren't a thing in C++ and even flexible array
+     * members are a language extension and beyond the realm of standard C++.
+     * Technically, sizeof data_ member is 0 and the intended way to use flexible
+     * array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
+     * and the compiler will consider alignment restrictions for the FAM.
+     *
+     */
+
+    StringData(StringData &&) = delete;
+    StringData & operator=(StringData &&) = delete;
+    StringData(const StringData &) = delete;
+    StringData & operator=(const StringData &) = delete;
+    ~StringData() = default;
+
+private:
+    StringData() = delete;
+
+    explicit StringData(size_type size)
+        : size_(size)
+    {
+    }
+
+public:
+    /**
+     * Allocate StringData on the (possibly) GC-managed heap and copy
+     * the contents of s to it.
+     */
+    static const StringData & make(std::string_view s);
+
+    /**
+     * Allocate StringData on the (possibly) GC-managed heap.
+     * @param size Length of the string (without the NUL terminator).
+     */
+    static StringData & alloc(size_t size);
+
+    size_t size() const
+    {
+        return size_;
+    }
+
+    char * data() noexcept
+    {
+        return data_;
+    }
+
+    const char * data() const noexcept
+    {
+        return data_;
+    }
+
+    const char * c_str() const noexcept
+    {
+        return data_;
+    }
+
+    constexpr std::string_view view() const noexcept
+    {
+        return std::string_view(data_, size_);
+    }
+
+    template<size_t N>
+    struct Static;
+
+    static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
+    {
+        auto & res =
+            *new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
+        std::memcpy(res.data_, s.data(), s.size());
+        res.data_[s.size()] = '\0';
+        return res;
+    }
+};
+
 namespace detail {
 
 /**
@@ -219,7 +311,7 @@ struct ValueBase
      */
     struct StringWithContext
     {
-        const char * c_str;
+        const StringData * str;
 
         /**
          * The type of the context itself.
@@ -234,7 +326,7 @@ struct ValueBase
          */
         struct Context
         {
-            using value_type = const char *;
+            using value_type = const StringData *;
             using size_type = std::size_t;
             using iterator = const value_type *;
 
@@ -273,7 +365,7 @@ struct ValueBase
             /**
              * @return null pointer when context.empty()
              */
-            static Context * fromBuilder(const NixStringContext & context);
+            static Context * fromBuilder(const NixStringContext & context, EvalMemory & mem);
         };
 
         /**
@@ -285,7 +377,7 @@ struct ValueBase
     struct Path
     {
         SourceAccessor * accessor;
-        const char * path;
+        const StringData * path;
     };
 
     struct Null
@@ -646,13 +738,13 @@ protected:
     void getStorage(StringWithContext & string) const noexcept
     {
         string.context = untagPointer<decltype(string.context)>(payload[0]);
-        string.c_str = std::bit_cast<const char *>(payload[1]);
+        string.str = std::bit_cast<const StringData *>(payload[1]);
     }
 
     void getStorage(Path & path) const noexcept
     {
         path.accessor = untagPointer<decltype(path.accessor)>(payload[0]);
-        path.path = std::bit_cast<const char *>(payload[1]);
+        path.path = std::bit_cast<const StringData *>(payload[1]);
     }
 
     void setStorage(NixInt integer) noexcept
@@ -697,7 +789,7 @@ protected:
 
     void setStorage(StringWithContext string) noexcept
     {
-        setUntaggablePayload<pdString>(string.context, string.c_str);
+        setUntaggablePayload<pdString>(string.context, string.str);
     }
 
     void setStorage(Path path) noexcept
@@ -1050,22 +1142,22 @@ public:
         setStorage(b);
     }
 
-    void mkStringNoCopy(const char * s, const Value::StringWithContext::Context * context = nullptr) noexcept
+    void mkStringNoCopy(const StringData & s, const Value::StringWithContext::Context * context = nullptr) noexcept
     {
-        setStorage(StringWithContext{.c_str = s, .context = context});
+        setStorage(StringWithContext{.str = &s, .context = context});
     }
 
     void mkString(std::string_view s);
 
-    void mkString(std::string_view s, const NixStringContext & context);
+    void mkString(std::string_view s, const NixStringContext & context, EvalMemory & mem);
 
-    void mkStringMove(const char * s, const NixStringContext & context);
+    void mkStringMove(const StringData & s, const NixStringContext & context, EvalMemory & mem);
 
     void mkPath(const SourcePath & path);
 
-    inline void mkPath(SourceAccessor * accessor, const char * path) noexcept
+    inline void mkPath(SourceAccessor * accessor, const StringData & path) noexcept
     {
-        setStorage(Path{.accessor = accessor, .path = path});
+        setStorage(Path{.accessor = accessor, .path = &path});
     }
 
     inline void mkNull() noexcept
@@ -1163,17 +1255,23 @@ public:
 
     SourcePath path() const
     {
-        return SourcePath(ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), pathStr()));
+        return SourcePath(
+            ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), std::string(pathStrView())));
     }
 
-    std::string_view string_view() const noexcept
+    const StringData & string_data() const noexcept
     {
-        return std::string_view{getStorage<StringWithContext>().c_str};
+        return *getStorage<StringWithContext>().str;
     }
 
     const char * c_str() const noexcept
     {
-        return getStorage<StringWithContext>().c_str;
+        return getStorage<StringWithContext>().str->data();
+    }
+
+    std::string_view string_view() const noexcept
+    {
+        return string_data().view();
     }
 
     const Value::StringWithContext::Context * context() const noexcept
@@ -1233,12 +1331,12 @@ public:
 
     const char * pathStr() const noexcept
     {
-        return getStorage<Path>().path;
+        return getStorage<Path>().path->c_str();
     }
 
     std::string_view pathStrView() const noexcept
     {
-        return std::string_view{getStorage<Path>().path};
+        return getStorage<Path>().path->view();
     }
 
     SourceAccessor * pathAccessor() const noexcept

src/libexpr/package.nix

@@ -70,11 +70,6 @@ mkMesonLibrary (finalAttrs: {
     nix-util
     nix-store
     nix-fetchers
-  ]
-  ++ finalAttrs.passthru.externalPropagatedBuildInputs;
-
-  # Hack for sake of the dev shell
-  passthru.externalPropagatedBuildInputs = [
     boost
     nlohmann_json
   ]

src/libexpr/parser.y

@@ -126,26 +126,26 @@ static Expr * makeCall(Exprs & exprs, PosIdx pos, Expr * fn, Expr * arg) {
 
 %define api.value.type variant
 
-%type <nix::Expr *> start expr expr_function expr_if expr_op
-%type <nix::Expr *> expr_select expr_simple expr_app
-%type <nix::Expr *> expr_pipe_from expr_pipe_into
+%type <Expr *> start expr expr_function expr_if expr_op
+%type <Expr *> expr_select expr_simple expr_app
+%type <Expr *> expr_pipe_from expr_pipe_into
 %type <std::vector<Expr *>> list
-%type <nix::ExprAttrs *> binds binds1
-%type <nix::FormalsBuilder> formals formal_set
-%type <nix::Formal> formal
-%type <std::vector<nix::AttrName>> attrpath
-%type <std::vector<std::pair<nix::AttrName, nix::PosIdx>>> attrs
-%type <std::vector<std::pair<nix::PosIdx, nix::Expr *>>> string_parts_interpolated
-%type <std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>>> ind_string_parts
-%type <nix::Expr *> path_start
-%type <std::variant<nix::Expr *, std::string_view>> string_parts string_attr
-%type <nix::StringToken> attr
-%token <nix::StringToken> ID
-%token <nix::StringToken> STR IND_STR
-%token <nix::NixInt> INT_LIT
-%token <nix::NixFloat> FLOAT_LIT
-%token <nix::StringToken> PATH HPATH SPATH PATH_END
-%token <nix::StringToken> URI
+%type <ExprAttrs *> binds binds1
+%type <FormalsBuilder> formals formal_set
+%type <Formal> formal
+%type <std::vector<AttrName>> attrpath
+%type <std::vector<std::pair<AttrName, PosIdx>>> attrs
+%type <std::vector<std::pair<PosIdx, Expr *>>> string_parts_interpolated
+%type <std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>>> ind_string_parts
+%type <Expr *> path_start
+%type <std::variant<Expr *, std::string_view>> string_parts string_attr
+%type <StringToken> attr
+%token <StringToken> ID
+%token <StringToken> STR IND_STR
+%token <NixInt> INT_LIT
+%token <NixFloat> FLOAT_LIT
+%token <StringToken> PATH HPATH SPATH PATH_END
+%token <StringToken> URI
 %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW
 %token PIPE_FROM PIPE_INTO /* <| and |> */
 %token DOLLAR_CURLY /* == ${ */
@@ -186,7 +186,7 @@ expr_function
   | formal_set ':' expr_function[body]
     {
       state->validateFormals($formal_set);
-      auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, std::move($formal_set), $body);
+      auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, $formal_set, $body);
       $$ = me;
       SET_DOC_POS(me, @1);
     }
@@ -194,7 +194,7 @@ expr_function
     {
       auto arg = state->symbols.create($ID);
       state->validateFormals($formal_set, CUR_POS, arg);
-      auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, std::move($formal_set), $body);
+      auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, $formal_set, $body);
       $$ = me;
       SET_DOC_POS(me, @1);
     }
@@ -202,7 +202,7 @@ expr_function
     {
       auto arg = state->symbols.create($ID);
       state->validateFormals($formal_set, CUR_POS, arg);
-      auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, std::move($formal_set), $body);
+      auto me = state->exprs.add<ExprLambda>(state->positions, state->exprs.alloc, CUR_POS, arg, $formal_set, $body);
       $$ = me;
       SET_DOC_POS(me, @1);
     }
@@ -251,7 +251,7 @@ expr_op
   | expr_op OR expr_op { $$ = state->exprs.add<ExprOpOr>(state->at(@2), $1, $3); }
   | expr_op IMPL expr_op { $$ = state->exprs.add<ExprOpImpl>(state->at(@2), $1, $3); }
   | expr_op UPDATE expr_op { $$ = state->exprs.add<ExprOpUpdate>(state->at(@2), $1, $3); }
-  | expr_op '?' attrpath { $$ = state->exprs.add<ExprOpHasAttr>(state->exprs.alloc, $1, std::move($3)); }
+  | expr_op '?' attrpath { $$ = state->exprs.add<ExprOpHasAttr>(state->exprs.alloc, $1, $3); }
   | expr_op '+' expr_op
     { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, state->at(@2), false, {{state->at(@1), $1}, {state->at(@3), $3}}); }
   | expr_op '-' expr_op { $$ = state->exprs.add<ExprCall>(state->at(@2), state->exprs.add<ExprVar>(state->s.sub), {$1, $3}); }
@@ -272,9 +272,9 @@ expr_app
 
 expr_select
   : expr_simple '.' attrpath
-    { $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, std::move($3), nullptr); }
+    { $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, $3, nullptr); }
   | expr_simple '.' attrpath OR_KW expr_select
-    { $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, std::move($3), $5); $5->warnIfCursedOr(state->symbols, state->positions); }
+    { $$ = state->exprs.add<ExprSelect>(state->exprs.alloc, CUR_POS, $1, $3, $5); $5->warnIfCursedOr(state->symbols, state->positions); }
   | /* Backwards compatibility: because Nixpkgs has a function named ‘or’,
        allow stuff like ‘map or [...]’. This production is problematic (see
        https://github.com/NixOS/nix/issues/11118) and will be refactored in the
@@ -304,12 +304,12 @@ expr_simple
       $2);
   }
   | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE {
-      $$ = state->stripIndentation(CUR_POS, std::move($2));
+      $$ = state->stripIndentation(CUR_POS, $2);
   }
   | path_start PATH_END
   | path_start string_parts_interpolated PATH_END {
       $2.insert($2.begin(), {state->at(@1), $1});
-      $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, false, std::move($2));
+      $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, false, $2);
   }
   | SPATH {
       std::string_view path($1.p + 1, $1.l - 2);
@@ -338,12 +338,12 @@ expr_simple
     { $2->pos = CUR_POS; $$ = $2; }
   | '{' '}'
     { $$ = state->exprs.add<ExprAttrs>(CUR_POS); }
-  | '[' list ']' { $$ = state->exprs.add<ExprList>(state->exprs.alloc, std::move($2)); }
+  | '[' list ']' { $$ = state->exprs.add<ExprList>(state->exprs.alloc, $2); }
   ;
 
 string_parts
   : STR { $$ = $1; }
-  | string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, std::move($1)); }
+  | string_parts_interpolated { $$ = state->exprs.add<ExprConcatStrings>(state->exprs.alloc, CUR_POS, true, $1); }
   | { $$ = std::string_view(); }
   ;
 
@@ -425,7 +425,7 @@ binds1
       if (!$accum->inheritFromExprs)
           $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
       $accum->inheritFromExprs->push_back($expr);
-      auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1);
+      auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1);
       for (auto & [i, iPos] : $attrs) {
           if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
               state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);

src/libexpr/primops.cc

@@ -5,6 +5,7 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/gc-small-vector.hh"
 #include "nix/expr/json-to-value.hh"
+#include "nix/expr/static-string-data.hh"
 #include "nix/store/globals.hh"
 #include "nix/store/names.hh"
 #include "nix/store/path-references.hh"
@@ -39,6 +40,12 @@
 
 namespace nix {
 
+RegisterPrimOp::PrimOps & RegisterPrimOp::primOps()
+{
+    static RegisterPrimOp::PrimOps primOps;
+    return primOps;
+}
+
 /*************************************************************
  * Miscellaneous
  *************************************************************/
@@ -221,7 +228,8 @@ void derivationToValue(
             path2,
             {
                 NixStringContextElem::DrvDeep{.drvPath = storePath},
-            });
+            },
+            state.mem);
     attrs.alloc(state.s.name).mkString(drv.env["name"]);
 
     auto list = state.buildList(drv.outputs.size());
@@ -487,34 +495,34 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Valu
     state.forceValue(*args[0], pos);
     switch (args[0]->type()) {
     case nInt:
-        v.mkStringNoCopy("int");
+        v.mkStringNoCopy("int"_sds);
         break;
     case nBool:
-        v.mkStringNoCopy("bool");
+        v.mkStringNoCopy("bool"_sds);
         break;
     case nString:
-        v.mkStringNoCopy("string");
+        v.mkStringNoCopy("string"_sds);
         break;
     case nPath:
-        v.mkStringNoCopy("path");
+        v.mkStringNoCopy("path"_sds);
         break;
     case nNull:
-        v.mkStringNoCopy("null");
+        v.mkStringNoCopy("null"_sds);
         break;
     case nAttrs:
-        v.mkStringNoCopy("set");
+        v.mkStringNoCopy("set"_sds);
         break;
     case nList:
-        v.mkStringNoCopy("list");
+        v.mkStringNoCopy("list"_sds);
         break;
     case nFunction:
-        v.mkStringNoCopy("lambda");
+        v.mkStringNoCopy("lambda"_sds);
         break;
     case nExternal:
         v.mkString(args[0]->external()->typeOf());
         break;
     case nFloat:
-        v.mkStringNoCopy("float");
+        v.mkStringNoCopy("float"_sds);
         break;
     case nThunk:
         unreachable();
@@ -1810,7 +1818,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName
             drvPathS,
             {
                 NixStringContextElem::DrvDeep{.drvPath = drvPath},
-            });
+            },
+            state.mem);
     for (auto & i : drv.outputs)
         mkOutputString(state, result, drvPath, i);
 
@@ -1863,7 +1872,7 @@ static void prim_toPath(EvalState & state, const PosIdx pos, Value ** args, Valu
     NixStringContext context;
     auto path =
         state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath");
-    v.mkString(path.path.abs(), context);
+    v.mkString(path.path.abs(), context, state.mem);
 }
 
 static RegisterPrimOp primop_toPath({
@@ -1906,7 +1915,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value ** args, V
     if (!settings.readOnlyMode)
         state.store->ensurePath(path2);
     context.insert(NixStringContextElem::Opaque{.path = path2});
-    v.mkString(path.abs(), context);
+    v.mkString(path.abs(), context, state.mem);
 }
 
 static RegisterPrimOp primop_storePath({
@@ -1988,7 +1997,8 @@ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value ** args,
     v.mkString(
         legacyBaseNameOf(*state.coerceToString(
             pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)),
-        context);
+        context,
+        state.mem);
 }
 
 static RegisterPrimOp primop_baseNameOf({
@@ -2024,11 +2034,11 @@ static void prim_dirOf(EvalState & state, const PosIdx pos, Value ** args, Value
             pos, *args[0], context, "while evaluating the first argument passed to 'builtins.dirOf'", false, false);
         auto pos = path->rfind('/');
         if (pos == path->npos)
-            v.mkStringMove(".", context);
+            v.mkStringMove("."_sds, context, state.mem);
         else if (pos == 0)
-            v.mkStringMove("/", context);
+            v.mkStringMove("/"_sds, context, state.mem);
         else
-            v.mkString(path->substr(0, pos), context);
+            v.mkString(path->substr(0, pos), context, state.mem);
     }
 }
 
@@ -2070,7 +2080,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Va
                 .path = std::move((StorePath &&) p),
             });
     }
-    v.mkString(s, context);
+    v.mkString(s, context, state.mem);
 }
 
 static RegisterPrimOp primop_readFile({
@@ -2309,10 +2319,10 @@ static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type ty
 
     static const Constants stringValues = []() {
         Constants res;
-        res.regular.mkStringNoCopy("regular");
-        res.directory.mkStringNoCopy("directory");
-        res.symlink.mkStringNoCopy("symlink");
-        res.unknown.mkStringNoCopy("unknown");
+        res.regular.mkStringNoCopy("regular"_sds);
+        res.directory.mkStringNoCopy("directory"_sds);
+        res.symlink.mkStringNoCopy("symlink"_sds);
+        res.unknown.mkStringNoCopy("unknown"_sds);
         return res;
     }();
 
@@ -2466,7 +2476,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value
     std::ostringstream out;
     NixStringContext context;
     printValueAsXML(state, true, false, *args[0], out, context, pos);
-    v.mkString(out.view(), context);
+    v.mkString(out.view(), context, state.mem);
 }
 
 static RegisterPrimOp primop_toXML({
@@ -2574,7 +2584,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu
     std::ostringstream out;
     NixStringContext context;
     printValueAsJSON(state, true, *args[0], pos, out, context);
-    v.mkString(out.view(), context);
+    v.mkString(out.view(), context, state.mem);
 }
 
 static RegisterPrimOp primop_toJSON({
@@ -4403,7 +4413,7 @@ static void prim_toString(EvalState & state, const PosIdx pos, Value ** args, Va
     NixStringContext context;
     auto s = state.coerceToString(
         pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false);
-    v.mkString(*s, context);
+    v.mkString(*s, context, state.mem);
 }
 
 static RegisterPrimOp primop_toString({
@@ -4463,7 +4473,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
     if (len == 0) {
         state.forceValue(*args[2], pos);
         if (args[2]->type() == nString) {
-            v.mkStringNoCopy("", args[2]->context());
+            v.mkStringNoCopy(""_sds, args[2]->context());
             return;
         }
     }
@@ -4476,7 +4486,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V
     auto s = state.coerceToString(
         pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring");
 
-    v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context);
+    v.mkString(NixUInt(start) >= s->size() ? "" : s->substr(start, _len), context, state.mem);
 }
 
 static RegisterPrimOp primop_substring({
@@ -4874,7 +4884,7 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value **
             "while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep");
     }
 
-    v.mkString(res, context);
+    v.mkString(res, context, state.mem);
 }
 
 static RegisterPrimOp primop_concatStringsSep({
@@ -4949,7 +4959,7 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** ar
         }
     }
 
-    v.mkString(res, context);
+    v.mkString(res, context, state.mem);
 }
 
 static RegisterPrimOp primop_replaceStrings({

src/libexpr/primops/context.cc

@@ -69,7 +69,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
         }
     }
 
-    v.mkString(*s, context2);
+    v.mkString(*s, context2, state.mem);
 }
 
 static RegisterPrimOp primop_unsafeDiscardOutputDependency(
@@ -137,7 +137,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
             context.begin()->raw)}),
     };
 
-    v.mkString(*s, context2);
+    v.mkString(*s, context2, state.mem);
 }
 
 static RegisterPrimOp primop_addDrvOutputDependencies(
@@ -321,7 +321,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg
         }
     }
 
-    v.mkString(orig, context);
+    v.mkString(orig, context, state.mem);
 }
 
 static RegisterPrimOp primop_appendContext({.name = "__appendContext", .arity = 2, .fun = prim_appendContext});

src/libexpr/primops/fetchTree.cc

@@ -317,77 +317,11 @@ static RegisterPrimOp primop_fetchTree({
             > }
             > ```
 
-      - `"tarball"`
-
-        Download a tar archive and extract it into the Nix store.
-        This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
-
-        - `url` (String, required)
-
-           > **Example**
-           >
-           > ```nix
-           > fetchTree {
-           >   type = "tarball";
-           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
-           > }
-           > ```
-
       - `"git"`
 
         Fetch a Git tree and copy it to the Nix store.
         This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
 
-        - `url` (String, required)
-
-          The URL formats supported are the same as for Git itself.
-
-          > **Example**
-          >
-          > ```nix
-          > fetchTree {
-          >   type = "git";
-          >   url = "git@github.com:NixOS/nixpkgs.git";
-          > }
-          > ```
-
-          > **Note**
-          >
-          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
-
-        - `ref` (String, optional)
-
-          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
-
-          Default: `"HEAD"`
-
-        - `rev` (String, optional)
-
-          A Git revision; a commit hash.
-
-          Default: the tip of `ref`
-
-        - `shallow` (Bool, optional)
-
-          Make a shallow clone when fetching the Git tree.
-          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
-
-          Default: `true`
-
-        - `submodules` (Bool, optional)
-
-          Also fetch submodules if available.
-
-          Default: `false`
-
-        - `lfs` (Bool, optional)
-
-          Fetch any [Git LFS](https://git-lfs.com/) files.
-
-          Default: `false`
-
         - `allRefs` (Bool, optional)
 
           By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
@@ -405,6 +339,26 @@ static RegisterPrimOp primop_fetchTree({
           If set, pass through the value to the output attribute set.
           Otherwise, generated from the fetched Git tree.
 
+        - `lfs` (Bool, optional)
+
+          Fetch any [Git LFS](https://git-lfs.com/) files.
+
+          Default: `false`
+
+        - `ref` (String, optional)
+
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+          Default: `"HEAD"`
+
+        - `rev` (String, optional)
+
+          A Git revision; a commit hash.
+
+          Default: the tip of `ref`
+
         - `revCount` (Integer, optional)
 
           Number of revisions in the history of the Git repository before the fetched commit.
@@ -412,6 +366,52 @@ static RegisterPrimOp primop_fetchTree({
           If set, pass through the value to the output attribute set.
           Otherwise, generated from the fetched Git tree.
 
+        - `shallow` (Bool, optional)
+
+          Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+          Default: `true`
+
+        - `submodules` (Bool, optional)
+
+          Also fetch submodules if available.
+
+          Default: `false`
+
+        - `url` (String, required)
+
+          The URL formats supported are the same as for Git itself.
+
+          > **Example**
+          >
+          > ```nix
+          > fetchTree {
+          >   type = "git";
+          >   url = "git@github.com:NixOS/nixpkgs.git";
+          > }
+          > ```
+
+          > **Note**
+          >
+          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
+
+      - `"tarball"`
+
+        Download a tar archive and extract it into the Nix store.
+        This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+
+        - `url` (String, required)
+
+           > **Example**
+           >
+           > ```nix
+           > fetchTree {
+           >   type = "tarball";
+           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+           > }
+           > ```
+
       The following input types are still subject to change:
 
       - `"path"`

src/libexpr/primops/fromTOML.cc

@@ -1,5 +1,6 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
+#include "nix/expr/static-string-data.hh"
 
 #include "expr-config-private.hh"
 
@@ -136,7 +137,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
                 normalizeDatetimeFormat(t);
 #endif
                 auto attrs = state.buildBindings(2);
-                attrs.alloc("_type").mkStringNoCopy("timestamp");
+                attrs.alloc("_type").mkStringNoCopy("timestamp"_sds);
                 std::ostringstream s;
                 s << t;
                 auto str = s.view();

src/libfetchers-tests/input.cc

@@ -0,0 +1,61 @@
+#include "nix/fetchers/fetch-settings.hh"
+#include "nix/fetchers/attrs.hh"
+#include "nix/fetchers/fetchers.hh"
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+namespace nix {
+
+using fetchers::Attr;
+
+struct InputFromAttrsTestCase
+{
+    fetchers::Attrs attrs;
+    std::string expectedUrl;
+    std::string description;
+    fetchers::Attrs expectedAttrs = attrs;
+};
+
+class InputFromAttrsTest : public ::testing::WithParamInterface<InputFromAttrsTestCase>, public ::testing::Test
+{};
+
+TEST_P(InputFromAttrsTest, attrsAreCorrectAndRoundTrips)
+{
+    fetchers::Settings fetchSettings;
+
+    const auto & testCase = GetParam();
+
+    auto input = fetchers::Input::fromAttrs(fetchSettings, fetchers::Attrs(testCase.attrs));
+
+    EXPECT_EQ(input.toAttrs(), testCase.expectedAttrs);
+    EXPECT_EQ(input.toURLString(), testCase.expectedUrl);
+
+    auto input2 = fetchers::Input::fromAttrs(fetchSettings, input.toAttrs());
+    EXPECT_EQ(input, input2);
+    EXPECT_EQ(input.toAttrs(), input2.toAttrs());
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    InputFromAttrs,
+    InputFromAttrsTest,
+    ::testing::Values(
+        // Test for issue #14429.
+        InputFromAttrsTestCase{
+            .attrs =
+                {
+                    {"url", Attr("git+ssh://git@github.com/NixOS/nixpkgs")},
+                    {"type", Attr("git")},
+                },
+            .expectedUrl = "git+ssh://git@github.com/NixOS/nixpkgs",
+            .description = "strips_git_plus_prefix",
+            .expectedAttrs =
+                {
+                    {"url", Attr("ssh://git@github.com/NixOS/nixpkgs")},
+                    {"type", Attr("git")},
+                },
+        }),
+    [](const ::testing::TestParamInfo<InputFromAttrsTestCase> & info) { return info.param.description; });
+
+} // namespace nix

src/libfetchers-tests/meson.build

@@ -42,6 +42,7 @@ sources = files(
   'access-tokens.cc',
   'git-utils.cc',
   'git.cc',
+  'input.cc',
   'nix_api_fetchers.cc',
   'public-key.cc',
 )

src/libfetchers/git-utils.cc

@@ -1328,13 +1328,18 @@ std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules
     return result;
 }
 
-ref<GitRepo> getTarballCache()
-{
-    static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache";
+namespace fetchers {
 
-    return GitRepo::openRepo(repoDir, true, true);
+ref<GitRepo> Settings::getTarballCache() const
+{
+    auto tarballCache(_tarballCache.lock());
+    if (!*tarballCache)
+        *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true);
+    return ref<GitRepo>(*tarballCache);
 }
 
+} // namespace fetchers
+
 GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path)
 {
     static Sync<std::map<std::filesystem::path, WorkdirInfo>> _cache;

src/libfetchers/git.cc

@@ -168,8 +168,6 @@ struct GitInputScheme : InputScheme
             return {};
 
         auto url2(url);
-        if (hasPrefix(url2.scheme, "git+"))
-            url2.scheme = std::string(url2.scheme, 4);
         url2.query.clear();
 
         Attrs attrs;

src/libfetchers/github.cc

@@ -270,7 +270,7 @@ struct GitArchiveInputScheme : InputScheme
             if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) {
                 auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
                 auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
-                if (getTarballCache()->hasObject(treeHash))
+                if (input.settings->getTarballCache()->hasObject(treeHash))
                     return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}};
                 else
                     debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
@@ -290,7 +290,7 @@ struct GitArchiveInputScheme : InputScheme
             *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string()));
 
         TarArchive archive{*source};
-        auto tarballCache = getTarballCache();
+        auto tarballCache = input.settings->getTarballCache();
         auto parseSink = tarballCache->getFileSystemObjectSink();
         auto lastModified = unpackTarfileToSink(archive, *parseSink);
         auto tree = parseSink->flush();
@@ -324,7 +324,8 @@ struct GitArchiveInputScheme : InputScheme
 #endif
         input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified));
 
-        auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
+        auto accessor =
+            input.settings->getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»");
 
         return {accessor, input};
     }

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -11,6 +11,12 @@
 
 #include <sys/types.h>
 
+namespace nix {
+
+struct GitRepo;
+
+}
+
 namespace nix::fetchers {
 
 struct Cache;
@@ -125,8 +131,12 @@ struct Settings : public Config
 
     ref<Cache> getCache() const;
 
+    ref<GitRepo> getTarballCache() const;
+
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;
+
+    mutable Sync<std::shared_ptr<GitRepo>> _tarballCache;
 };
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/git-utils.hh

@@ -120,8 +120,6 @@ struct GitRepo
     virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0;
 };
 
-ref<GitRepo> getTarballCache();
-
 // A helper to ensure that the `git_*_free` functions get called.
 template<auto del>
 struct Deleter

src/libfetchers/tarball.cc

@@ -136,11 +136,11 @@ static DownloadTarballResult downloadTarball_(
             .treeHash = treeHash,
             .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"),
             .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"),
-            .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix),
+            .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix),
         };
     };
 
-    if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash")))
+    if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash")))
         cached.reset();
 
     if (cached && !cached->expired)
@@ -179,7 +179,7 @@ static DownloadTarballResult downloadTarball_(
         TarArchive{path};
     })
                                                                                     : TarArchive{*source};
-    auto tarballCache = getTarballCache();
+    auto tarballCache = settings.getTarballCache();
     auto parseSink = tarballCache->getFileSystemObjectSink();
     auto lastModified = unpackTarfileToSink(archive, *parseSink);
     auto tree = parseSink->flush();
@@ -398,7 +398,9 @@ struct TarballInputScheme : CurlInputScheme
 
         input.attrs.insert_or_assign(
             "narHash",
-            getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true));
+            input.settings->getTarballCache()
+                ->treeHashToNarHash(*input.settings, result.treeHash)
+                .to_string(HashFormat::SRI, true));
 
         return {result.accessor, input};
     }

src/libmain/progress-bar.cc

@@ -467,8 +467,6 @@ public:
 
     std::string getStatus(State & state)
     {
-        auto MiB = 1024.0 * 1024.0;
-
         std::string res;
 
         auto renderActivity =
@@ -516,6 +514,65 @@ public:
                 return s;
             };
 
+        auto renderSizeActivity = [&](ActivityType type, const std::string & itemFmt = "%s") {
+            auto & act = state.activitiesByType[type];
+            uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed;
+            for (auto & j : act.its) {
+                done += j.second->done;
+                expected += j.second->expected;
+                running += j.second->running;
+                failed += j.second->failed;
+            }
+
+            expected = std::max(expected, act.expected);
+
+            std::optional<SizeUnit> commonUnit;
+            std::string s;
+
+            if (running || done || expected || failed) {
+                if (running)
+                    if (expected != 0) {
+                        commonUnit = getCommonSizeUnit({(int64_t) running, (int64_t) done, (int64_t) expected});
+                        s =
+                            fmt(ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL "/%s",
+                                commonUnit ? renderSizeWithoutUnit(running, *commonUnit) : renderSize(running),
+                                commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done),
+                                commonUnit ? renderSizeWithoutUnit(expected, *commonUnit) : renderSize(expected));
+                    } else {
+                        commonUnit = getCommonSizeUnit({(int64_t) running, (int64_t) done});
+                        s =
+                            fmt(ANSI_BLUE "%s" ANSI_NORMAL "/" ANSI_GREEN "%s" ANSI_NORMAL,
+                                commonUnit ? renderSizeWithoutUnit(running, *commonUnit) : renderSize(running),
+                                commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done));
+                    }
+                else if (expected != done)
+                    if (expected != 0) {
+                        commonUnit = getCommonSizeUnit({(int64_t) done, (int64_t) expected});
+                        s =
+                            fmt(ANSI_GREEN "%s" ANSI_NORMAL "/%s",
+                                commonUnit ? renderSizeWithoutUnit(done, *commonUnit) : renderSize(done),
+                                commonUnit ? renderSizeWithoutUnit(expected, *commonUnit) : renderSize(expected));
+                    } else {
+                        commonUnit = getSizeUnit(done);
+                        s = fmt(ANSI_GREEN "%s" ANSI_NORMAL, renderSizeWithoutUnit(done, *commonUnit));
+                    }
+                else {
+                    commonUnit = getSizeUnit(done);
+                    s = fmt(done ? ANSI_GREEN "%s" ANSI_NORMAL : "%s", renderSizeWithoutUnit(done, *commonUnit));
+                }
+
+                if (commonUnit)
+                    s = fmt("%s %siB", s, getSizeUnitSuffix(*commonUnit));
+
+                s = fmt(itemFmt, s);
+
+                if (failed)
+                    s += fmt(" (" ANSI_RED "%s failed" ANSI_NORMAL ")", renderSize(failed));
+            }
+
+            return s;
+        };
+
         auto showActivity =
             [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) {
                 auto s = renderActivity(type, itemFmt, numberFmt, unit);
@@ -529,7 +586,7 @@ public:
         showActivity(actBuilds, "%s built");
 
         auto s1 = renderActivity(actCopyPaths, "%s copied");
-        auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB);
+        auto s2 = renderSizeActivity(actCopyPath);
 
         if (!s1.empty() || !s2.empty()) {
             if (!res.empty())
@@ -545,12 +602,12 @@ public:
             }
         }
 
-        showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB);
+        renderSizeActivity(actFileTransfer, "%s DL");
 
         {
             auto s = renderActivity(actOptimiseStore, "%s paths optimised");
             if (s != "") {
-                s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked);
+                s += fmt(", %s / %d inodes freed", renderSize(state.bytesLinked), state.filesLinked);
                 if (!res.empty())
                     res += ", ";
                 res += s;

src/libstore-test-support/include/nix/store/tests/protocol.hh

@@ -6,6 +6,7 @@
 
 #include "nix/store/tests/libstore.hh"
 #include "nix/util/tests/characterization.hh"
+#include "nix/util/tests/json-characterization.hh"
 
 namespace nix {
 
@@ -16,12 +17,30 @@ class ProtoTest : public CharacterizationTest
 
     std::filesystem::path goldenMaster(std::string_view testStem) const override
     {
-        return unitTestData / (std::string{testStem + ".bin"});
+        return unitTestData / testStem;
     }
 
 public:
     Path storeDir = "/nix/store";
     StoreDirConfig store{storeDir};
+
+    /**
+     * Golden test for `T` JSON reading
+     */
+    template<typename T>
+    void readJsonTest(PathView testStem, const T & expected)
+    {
+        nix::readJsonTest(*this, testStem, expected);
+    }
+
+    /**
+     * Golden test for `T` JSON write
+     */
+    template<typename T>
+    void writeJsonTest(PathView testStem, const T & decoded)
+    {
+        nix::writeJsonTest(*this, testStem, decoded);
+    }
 };
 
 template<class Proto, const char * protocolDir>
@@ -34,7 +53,7 @@ public:
     template<typename T>
     void readProtoTest(PathView testStem, typename Proto::Version version, T expected)
     {
-        CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
+        CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) {
             T got = ({
                 StringSource from{encoded};
                 Proto::template Serialise<T>::read(
@@ -55,7 +74,7 @@ public:
     template<typename T>
     void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded)
     {
-        CharacterizationTest::writeTest(testStem, [&]() {
+        CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() {
             StringSink to;
             Proto::template Serialise<T>::write(
                 this->store,
@@ -69,14 +88,25 @@ public:
     }
 };
 
-#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \
-    TEST_F(FIXTURE, NAME##_read)                                             \
-    {                                                                        \
-        readProtoTest(STEM, VERSION, VALUE);                                 \
-    }                                                                        \
-    TEST_F(FIXTURE, NAME##_write)                                            \
-    {                                                                        \
-        writeProtoTest(STEM, VERSION, VALUE);                                \
+#define VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_read)                                                     \
+    {                                                                                \
+        readProtoTest(STEM, VERSION, VALUE);                                         \
+    }                                                                                \
+    TEST_F(FIXTURE, NAME##_write)                                                    \
+    {                                                                                \
+        writeProtoTest(STEM, VERSION, VALUE);                                        \
+    }
+
+#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE)     \
+    VERSIONED_CHARACTERIZATION_TEST_NO_JSON(FIXTURE, NAME, STEM, VERSION, VALUE) \
+    TEST_F(FIXTURE, NAME##_json_read)                                            \
+    {                                                                            \
+        readJsonTest(STEM, VALUE);                                               \
+    }                                                                            \
+    TEST_F(FIXTURE, NAME##_json_write)                                           \
+    {                                                                            \
+        writeJsonTest(STEM, VALUE);                                              \
     }
 
 } // namespace nix

src/libstore-tests/common-protocol.cc

@@ -3,6 +3,7 @@
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
+#include "nix/util/json-utils.hh"
 #include "nix/store/common-protocol.hh"
 #include "nix/store/common-protocol-impl.hh"
 #include "nix/store/build-result.hh"
@@ -22,7 +23,7 @@ public:
     template<typename T>
     void readProtoTest(PathView testStem, const T & expected)
     {
-        CharacterizationTest::readTest(testStem, [&](const auto & encoded) {
+        CharacterizationTest::readTest(std::string{testStem + ".bin"}, [&](const auto & encoded) {
             T got = ({
                 StringSource from{encoded};
                 CommonProto::Serialise<T>::read(store, CommonProto::ReadConn{.from = from});
@@ -38,7 +39,7 @@ public:
     template<typename T>
     void writeProtoTest(PathView testStem, const T & decoded)
     {
-        CharacterizationTest::writeTest(testStem, [&]() -> std::string {
+        CharacterizationTest::writeTest(std::string{testStem + ".bin"}, [&]() -> std::string {
             StringSink to;
             CommonProto::Serialise<T>::write(store, CommonProto::WriteConn{.to = to}, decoded);
             return to.s;
@@ -54,6 +55,14 @@ public:
     TEST_F(CommonProtoTest, NAME##_write)        \
     {                                            \
         writeProtoTest(STEM, VALUE);             \
+    }                                            \
+    TEST_F(CommonProtoTest, NAME##_json_read)    \
+    {                                            \
+        readJsonTest(STEM, VALUE);               \
+    }                                            \
+    TEST_F(CommonProtoTest, NAME##_json_write)   \
+    {                                            \
+        writeJsonTest(STEM, VALUE);              \
     }
 
 CHARACTERIZATION_TEST(

src/libstore-tests/data/common-protocol/content-address.json

@@ -0,0 +1,26 @@
+[
+  {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "+Xc9Ll6mcPltwaewrk/BAQ56Y3G5T//wzhKUc0zrYu0="
+    },
+    "method": "text"
+  },
+  {
+    "hash": {
+      "algorithm": "sha1",
+      "format": "base64",
+      "hash": "gGemBoenViNZM3hiwqns/Fgzqwo="
+    },
+    "method": "flat"
+  },
+  {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  }
+]

src/libstore-tests/data/common-protocol/drv-output.json

@@ -0,0 +1,4 @@
+[
+  "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+  "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux"
+]

src/libstore-tests/data/common-protocol/optional-content-address.json

@@ -0,0 +1,11 @@
+[
+  null,
+  {
+    "hash": {
+      "algorithm": "sha1",
+      "format": "base64",
+      "hash": "gGemBoenViNZM3hiwqns/Fgzqwo="
+    },
+    "method": "flat"
+  }
+]

src/libstore-tests/data/common-protocol/optional-store-path.json

@@ -0,0 +1,4 @@
+[
+  null,
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar"
+]

src/libstore-tests/data/common-protocol/realisation-with-deps.json

@@ -0,0 +1,13 @@
+[
+  {
+    "dependentRealisations": {
+      "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"
+    },
+    "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+    "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+    "signatures": [
+      "asdf",
+      "qwer"
+    ]
+  }
+]

src/libstore-tests/data/common-protocol/realisation.json

@@ -0,0 +1,17 @@
+[
+  {
+    "dependentRealisations": {},
+    "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+    "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+    "signatures": []
+  },
+  {
+    "dependentRealisations": {},
+    "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+    "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+    "signatures": [
+      "asdf",
+      "qwer"
+    ]
+  }
+]

src/libstore-tests/data/common-protocol/set.json

@@ -0,0 +1,22 @@
+[
+  [],
+  [
+    ""
+  ],
+  [
+    "",
+    "bar",
+    "foo"
+  ],
+  [
+    [],
+    [
+      ""
+    ],
+    [
+      "",
+      "1",
+      "2"
+    ]
+  ]
+]

src/libstore-tests/data/common-protocol/store-path.json

@@ -0,0 +1,4 @@
+[
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar"
+]

src/libstore-tests/data/common-protocol/string.json

@@ -0,0 +1,7 @@
+[
+  "",
+  "hi",
+  "white rabbit",
+  "大白兔",
+  "oh no "
+]

src/libstore-tests/data/common-protocol/vector.json

@@ -0,0 +1,22 @@
+[
+  [],
+  [
+    ""
+  ],
+  [
+    "",
+    "foo",
+    "bar"
+  ],
+  [
+    [],
+    [
+      ""
+    ],
+    [
+      "",
+      "1",
+      "2"
+    ]
+  ]
+]

src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json

@@ -69,7 +69,8 @@
     "outputChecks": {
       "bin": {
         "disallowedReferences": [
-          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
+          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+          "dev"
         ],
         "disallowedRequisites": [
           "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
@@ -84,7 +85,8 @@
           "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
         ],
         "allowedRequisites": [
-          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
+          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+          "bin"
         ]
       }
     },

src/libstore-tests/data/derivation/ca/advanced-attributes.json

@@ -11,9 +11,9 @@
     "__sandboxProfile": "sandcastle",
     "allowSubstitutes": "",
     "allowedReferences": "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9",
-    "allowedRequisites": "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+    "allowedRequisites": "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z bin",
     "builder": "/bin/bash",
-    "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+    "disallowedReferences": "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g dev",
     "disallowedRequisites": "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8",
     "exportReferencesGraph": "refs1 /164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9 refs2 /nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv",
     "impureEnvVars": "UNICORN",

src/libstore-tests/data/derivation/ca/all_set.json

@@ -23,10 +23,12 @@
         "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
       ],
       "allowedRequisites": [
-        "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
+        "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+        "bin"
       ],
       "disallowedReferences": [
-        "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
+        "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+        "dev"
       ],
       "disallowedRequisites": [
         "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"

src/libstore-tests/data/derivation/ca/structuredAttrs_all_set.json

@@ -23,7 +23,8 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g"
+          "/0nyw57wm2iicnm9rglvjmbci3ikmcp823czdqdzdcgsnnwqps71g",
+          "dev"
         ],
         "disallowedRequisites": [
           "/07f301yqyz8c6wf6bbbavb2q39j4n8kmcly1s09xadyhgy6x2wr8"
@@ -46,7 +47,8 @@
           "/164j69y6zir9z0339n8pjigg3rckinlr77bxsavzizdaaljb7nh9"
         ],
         "allowedRequisites": [
-          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z"
+          "/0nr45p69vn6izw9446wsh9bng9nndhvn19kpsm4n96a5mycw0s4z",
+          "bin"
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json

@@ -5,9 +5,9 @@
   ],
   "builder": "/bin/bash",
   "env": {
-    "bin": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin",
-    "dev": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev",
-    "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
+    "bin": "/nix/store/cnpasdljgkhnwaf78cf3qygcp4qbki1c-advanced-attributes-structured-attrs-bin",
+    "dev": "/nix/store/ijq6mwpa9jbnpnl33qldfqihrr38kprx-advanced-attributes-structured-attrs-dev",
+    "out": "/nix/store/h1vh648d3p088kdimy0r8ngpfx7c3nzw-advanced-attributes-structured-attrs"
   },
   "inputs": {
     "drvs": {
@@ -33,13 +33,13 @@
   "name": "advanced-attributes-structured-attrs",
   "outputs": {
     "bin": {
-      "path": "33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin"
+      "path": "cnpasdljgkhnwaf78cf3qygcp4qbki1c-advanced-attributes-structured-attrs-bin"
     },
     "dev": {
-      "path": "wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev"
+      "path": "ijq6mwpa9jbnpnl33qldfqihrr38kprx-advanced-attributes-structured-attrs-dev"
     },
     "out": {
-      "path": "7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs"
+      "path": "h1vh648d3p088kdimy0r8ngpfx7c3nzw-advanced-attributes-structured-attrs"
     }
   },
   "structuredAttrs": {
@@ -66,7 +66,8 @@
     "outputChecks": {
       "bin": {
         "disallowedReferences": [
-          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
+          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+          "dev"
         ],
         "disallowedRequisites": [
           "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
@@ -81,7 +82,8 @@
           "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
         ],
         "allowedRequisites": [
-          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
+          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+          "bin"
         ]
       }
     },

src/libstore-tests/data/derivation/ia/advanced-attributes.json

@@ -11,14 +11,14 @@
     "__sandboxProfile": "sandcastle",
     "allowSubstitutes": "",
     "allowedReferences": "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo",
-    "allowedRequisites": "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+    "allowedRequisites": "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev bin",
     "builder": "/bin/bash",
-    "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+    "disallowedReferences": "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar dev",
     "disallowedRequisites": "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev",
     "exportReferencesGraph": "refs1 /nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo refs2 /nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv",
     "impureEnvVars": "UNICORN",
     "name": "advanced-attributes",
-    "out": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes",
+    "out": "/nix/store/ymqmybkq5j4nd1xplw6ccdpbjnfi017v-advanced-attributes",
     "preferLocalBuild": "1",
     "requiredSystemFeatures": "rainbow uid-range",
     "system": "my-system"
@@ -47,7 +47,7 @@
   "name": "advanced-attributes",
   "outputs": {
     "out": {
-      "path": "wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes"
+      "path": "ymqmybkq5j4nd1xplw6ccdpbjnfi017v-advanced-attributes"
     }
   },
   "system": "my-system",

src/libstore-tests/data/derivation/ia/all_set.json

@@ -23,10 +23,12 @@
         "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
       ],
       "allowedRequisites": [
-        "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
+        "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+        "bin"
       ],
       "disallowedReferences": [
-        "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
+        "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+        "dev"
       ],
       "disallowedRequisites": [
         "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"

src/libstore-tests/data/derivation/ia/structuredAttrs_all_set.json

@@ -23,7 +23,8 @@
         "allowedReferences": null,
         "allowedRequisites": null,
         "disallowedReferences": [
-          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar"
+          "/nix/store/r5cff30838majxk5mp3ip2diffi8vpaj-bar",
+          "dev"
         ],
         "disallowedRequisites": [
           "/nix/store/9b61w26b4avv870dw0ymb6rw4r1hzpws-bar-dev"
@@ -46,7 +47,8 @@
           "/nix/store/p0hax2lzvjpfc2gwkk62xdglz0fcqfzn-foo"
         ],
         "allowedRequisites": [
-          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev"
+          "/nix/store/z0rjzy29v9k5qa4nqpykrbzirj7sd43v-foo-dev",
+          "bin"
         ],
         "disallowedReferences": [],
         "disallowedRequisites": [],

src/libstore-tests/data/nar-info/impure.json

@@ -9,9 +9,17 @@
   },
   "compression": "xz",
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
-  "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "downloadHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "downloadSize": 4029176,
-  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "narSize": 34878,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",

src/libstore-tests/data/nar-info/pure.json

@@ -7,7 +7,11 @@
     },
     "method": "nar"
   },
-  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "narSize": 34878,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",

src/libstore-tests/data/path-info/empty_impure.json

@@ -1,7 +1,11 @@
 {
   "ca": null,
   "deriver": null,
-  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "narSize": 0,
   "references": [],
   "registrationTime": null,

src/libstore-tests/data/path-info/empty_pure.json

@@ -1,6 +1,10 @@
 {
   "ca": null,
-  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "narSize": 0,
   "references": [],
   "version": 2

src/libstore-tests/data/path-info/impure.json

@@ -8,7 +8,11 @@
     "method": "nar"
   },
   "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
-  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "narSize": 34878,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",

src/libstore-tests/data/path-info/pure.json

@@ -7,7 +7,11 @@
     },
     "method": "nar"
   },
-  "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=",
+  "narHash": {
+    "algorithm": "sha256",
+    "format": "base64",
+    "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+  },
   "narSize": 34878,
   "references": [
     "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",

src/libstore-tests/data/serve-protocol/build-result-2.2.json

@@ -0,0 +1,28 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "NotDeterministic",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "builtOutputs": {},
+    "startTime": 0,
+    "status": "Built",
+    "stopTime": 0,
+    "success": true,
+    "timesBuilt": 0
+  }
+]

src/libstore-tests/data/serve-protocol/build-result-2.3.json

@@ -0,0 +1,28 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": true,
+    "startTime": 30,
+    "status": "NotDeterministic",
+    "stopTime": 50,
+    "success": false,
+    "timesBuilt": 3
+  },
+  {
+    "builtOutputs": {},
+    "startTime": 30,
+    "status": "Built",
+    "stopTime": 50,
+    "success": true,
+    "timesBuilt": 0
+  }
+]

src/libstore-tests/data/serve-protocol/build-result-2.6.json

@@ -0,0 +1,41 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": true,
+    "startTime": 30,
+    "status": "NotDeterministic",
+    "stopTime": 50,
+    "success": false,
+    "timesBuilt": 3
+  },
+  {
+    "builtOutputs": {
+      "bar": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+        "signatures": []
+      },
+      "foo": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+        "signatures": []
+      }
+    },
+    "startTime": 30,
+    "status": "Built",
+    "stopTime": 50,
+    "success": true,
+    "timesBuilt": 1
+  }
+]

src/libstore-tests/data/serve-protocol/content-address.json

@@ -0,0 +1,26 @@
+[
+  {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "+Xc9Ll6mcPltwaewrk/BAQ56Y3G5T//wzhKUc0zrYu0="
+    },
+    "method": "text"
+  },
+  {
+    "hash": {
+      "algorithm": "sha1",
+      "format": "base64",
+      "hash": "gGemBoenViNZM3hiwqns/Fgzqwo="
+    },
+    "method": "flat"
+  },
+  {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  }
+]

src/libstore-tests/data/serve-protocol/drv-output.json

@@ -0,0 +1,4 @@
+[
+  "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+  "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux"
+]

src/libstore-tests/data/serve-protocol/optional-content-address.json

@@ -0,0 +1,11 @@
+[
+  null,
+  {
+    "hash": {
+      "algorithm": "sha1",
+      "format": "base64",
+      "hash": "gGemBoenViNZM3hiwqns/Fgzqwo="
+    },
+    "method": "flat"
+  }
+]

src/libstore-tests/data/serve-protocol/optional-store-path.json

@@ -0,0 +1,4 @@
+[
+  null,
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar"
+]

src/libstore-tests/data/serve-protocol/realisation-with-deps.json

@@ -0,0 +1,13 @@
+[
+  {
+    "dependentRealisations": {
+      "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"
+    },
+    "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+    "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+    "signatures": [
+      "asdf",
+      "qwer"
+    ]
+  }
+]

src/libstore-tests/data/serve-protocol/realisation.json

@@ -0,0 +1,17 @@
+[
+  {
+    "dependentRealisations": {},
+    "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+    "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+    "signatures": []
+  },
+  {
+    "dependentRealisations": {},
+    "id": "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+    "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+    "signatures": [
+      "asdf",
+      "qwer"
+    ]
+  }
+]

src/libstore-tests/data/serve-protocol/set.json

@@ -0,0 +1,22 @@
+[
+  [],
+  [
+    ""
+  ],
+  [
+    "",
+    "bar",
+    "foo"
+  ],
+  [
+    [],
+    [
+      ""
+    ],
+    [
+      "",
+      "1",
+      "2"
+    ]
+  ]
+]

src/libstore-tests/data/serve-protocol/store-path.json

@@ -0,0 +1,4 @@
+[
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar"
+]

src/libstore-tests/data/serve-protocol/string.json

@@ -0,0 +1,7 @@
+[
+  "",
+  "hi",
+  "white rabbit",
+  "大白兔",
+  "oh no "
+]

src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.json

@@ -0,0 +1,34 @@
+[
+  {
+    "ca": null,
+    "deriver": null,
+    "narHash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+    },
+    "narSize": 34878,
+    "references": [],
+    "registrationTime": null,
+    "signatures": [],
+    "ultimate": false,
+    "version": 2
+  },
+  {
+    "ca": null,
+    "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "narHash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
+    },
+    "narSize": 34878,
+    "references": [
+      "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv"
+    ],
+    "registrationTime": null,
+    "signatures": [],
+    "ultimate": false,
+    "version": 2
+  }
+]

src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.json

@@ -0,0 +1,47 @@
+[
+  {
+    "ca": null,
+    "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "narHash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+    },
+    "narSize": 34878,
+    "references": [
+      "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv"
+    ],
+    "registrationTime": null,
+    "signatures": [],
+    "ultimate": false,
+    "version": 2
+  },
+  {
+    "ca": {
+      "hash": {
+        "algorithm": "sha256",
+        "format": "base64",
+        "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+      },
+      "method": "nar"
+    },
+    "deriver": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "narHash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="
+    },
+    "narSize": 34878,
+    "references": [
+      "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+      "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"
+    ],
+    "registrationTime": null,
+    "signatures": [
+      "fake-sig-1",
+      "fake-sig-2"
+    ],
+    "ultimate": false,
+    "version": 2
+  }
+]

src/libstore-tests/data/serve-protocol/vector.json

@@ -0,0 +1,22 @@
+[
+  [],
+  [
+    ""
+  ],
+  [
+    "",
+    "foo",
+    "bar"
+  ],
+  [
+    [],
+    [
+      ""
+    ],
+    [
+      "",
+      "1",
+      "2"
+    ]
+  ]
+]

src/libstore-tests/data/worker-protocol/build-mode.json

@@ -0,0 +1,5 @@
+[
+  0,
+  1,
+  2
+]

src/libstore-tests/data/worker-protocol/build-result-1.27.json

@@ -0,0 +1,28 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "NotDeterministic",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "builtOutputs": {},
+    "startTime": 0,
+    "status": "Built",
+    "stopTime": 0,
+    "success": true,
+    "timesBuilt": 0
+  }
+]

src/libstore-tests/data/worker-protocol/build-result-1.28.json

@@ -0,0 +1,41 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "NotDeterministic",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "builtOutputs": {
+      "bar": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+        "signatures": []
+      },
+      "foo": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+        "signatures": []
+      }
+    },
+    "startTime": 0,
+    "status": "Built",
+    "stopTime": 0,
+    "success": true,
+    "timesBuilt": 0
+  }
+]

src/libstore-tests/data/worker-protocol/build-result-1.29.json

@@ -0,0 +1,41 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": true,
+    "startTime": 30,
+    "status": "NotDeterministic",
+    "stopTime": 50,
+    "success": false,
+    "timesBuilt": 3
+  },
+  {
+    "builtOutputs": {
+      "bar": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+        "signatures": []
+      },
+      "foo": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+        "signatures": []
+      }
+    },
+    "startTime": 30,
+    "status": "Built",
+    "stopTime": 50,
+    "success": true,
+    "timesBuilt": 1
+  }
+]

src/libstore-tests/data/worker-protocol/build-result-1.37.json

@@ -0,0 +1,43 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": true,
+    "startTime": 30,
+    "status": "NotDeterministic",
+    "stopTime": 50,
+    "success": false,
+    "timesBuilt": 3
+  },
+  {
+    "builtOutputs": {
+      "bar": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!bar",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar",
+        "signatures": []
+      },
+      "foo": {
+        "dependentRealisations": {},
+        "id": "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!foo",
+        "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+        "signatures": []
+      }
+    },
+    "cpuSystem": 604000000,
+    "cpuUser": 500000000,
+    "startTime": 30,
+    "status": "Built",
+    "stopTime": 50,
+    "success": true,
+    "timesBuilt": 1
+  }
+]

src/libstore-tests/data/worker-protocol/content-address.json

@@ -0,0 +1,26 @@
+[
+  {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "+Xc9Ll6mcPltwaewrk/BAQ56Y3G5T//wzhKUc0zrYu0="
+    },
+    "method": "text"
+  },
+  {
+    "hash": {
+      "algorithm": "sha1",
+      "format": "base64",
+      "hash": "gGemBoenViNZM3hiwqns/Fgzqwo="
+    },
+    "method": "flat"
+  },
+  {
+    "hash": {
+      "algorithm": "sha256",
+      "format": "base64",
+      "hash": "EMIJ+giQ/gLIWoxmPKjno3zHZrxbGymgzGGyZvZBIdM="
+    },
+    "method": "nar"
+  }
+]

src/libstore-tests/data/worker-protocol/derived-path-1.29.json

@@ -0,0 +1,16 @@
+[
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+  {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "outputs": [
+      "*"
+    ]
+  },
+  {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "outputs": [
+      "x",
+      "y"
+    ]
+  }
+]

src/libstore-tests/data/worker-protocol/derived-path-1.30.json

@@ -0,0 +1,17 @@
+[
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo",
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv",
+  {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "outputs": [
+      "*"
+    ]
+  },
+  {
+    "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+    "outputs": [
+      "x",
+      "y"
+    ]
+  }
+]

src/libstore-tests/data/worker-protocol/drv-output.json

@@ -0,0 +1,4 @@
+[
+  "sha256:15e3c560894cbb27085cf65b5a2ecb18488c999497f4531b6907a7581ce6d527!baz",
+  "sha256:6f869f9ea2823bda165e06076fd0de4366dead2c0e8d2dbbad277d4f15c373f5!quux"
+]

src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.json

@@ -0,0 +1,27 @@
+[
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": false,
+    "path": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx",
+    "startTime": 0,
+    "status": "OutputRejected",
+    "stopTime": 0,
+    "success": false,
+    "timesBuilt": 0
+  },
+  {
+    "errorMsg": "no idea why",
+    "isNonDeterministic": true,
+    "path": {
+      "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv",
+      "outputs": [
+        "out"
+      ]
+    },
+    "startTime": 30,
+    "status": "NotDeterministic",
+    "stopTime": 50,
+    "success": false,
+    "timesBuilt": 3
+  }
+]

src/libstore-tests/data/worker-protocol/optional-content-address.json

@@ -0,0 +1,11 @@
+[
+  null,
+  {
+    "hash": {
+      "algorithm": "sha1",
+      "format": "base64",
+      "hash": "gGemBoenViNZM3hiwqns/Fgzqwo="
+    },
+    "method": "flat"
+  }
+]

src/libstore-tests/data/worker-protocol/optional-store-path.json

@@ -0,0 +1,4 @@
+[
+  null,
+  "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar"
+]

src/libstore-tests/data/worker-protocol/optional-trusted-flag.json

@@ -0,0 +1,5 @@
+[
+  null,
+  true,
+  false
+]