Move StringData to its own header

Now that it isn't just used for `Value`, it doesn't really belong in
there.

Rename `static-string-data.hh` to share the same prefix to keep them
close together when sorting lines, also.

meson.build

@@ -13,7 +13,6 @@ project(
 )
 
 # Internal Libraries
-subproject('libcmarkcpp')
 subproject('libutil')
 subproject('libstore')
 subproject('libfetchers')

packaging/dev-shell.nix

@@ -269,8 +269,6 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
       CXX_LD = "mold";
     };
 
-    dontUseCmakeConfigure = true;
-
     mesonFlags =
       map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents2.nix-util.mesonFlags)
       ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents2.nix-store.mesonFlags)

packaging/hydra.nix

@@ -174,6 +174,24 @@ rec {
 
   buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli);
 
+  # Toggles some settings for better coverage. Windows needs these
+  # library combinations, and Debian build Nix with GNU readline too.
+  buildReadlineNoMarkdown =
+    let
+      components = forAllSystems (
+        system:
+        nixpkgsFor.${system}.native.nixComponents2.overrideScope (
+          self: super: {
+            nix-cmd = super.nix-cmd.override {
+              enableMarkdown = false;
+              readlineFlavor = "readline";
+            };
+          }
+        )
+      );
+    in
+    forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName}));
+
   # Perl bindings for various platforms.
   perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings);
 

src/libcmarkcpp/LICENSE.md

@@ -1,17 +0,0 @@
-Copyright (c) 2008, Natacha Porté
-Copyright (c) 2011, Vicent Martí
-Copyright (c) 2014, Xavier Mendez, Devin Torres and the Hoedown authors
-Copyright (c) 2016--2023, Kristaps Dzonsons
-Copyright (c) 2025, Obsidian Systems
-
-Permission to use, copy, modify, and/or distribute this software for any purpose
-with or without fee is hereby granted, provided that the above copyright notice
-and this permission notice appear in all copies.
-
-THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
-FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
-TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
-THIS SOFTWARE.

src/libcmarkcpp/README.md

@@ -1,36 +0,0 @@
-# libcmarkcpp
-
-A C++ terminal renderer for CommonMark documents.
-
-## Overview
-
-libcmarkcpp provides a terminal renderer for CommonMark (Markdown) documents using the cmark library. It renders formatted, colored output suitable for display in ANSI-capable terminals.
-
-## Features
-
-- ANSI color styling and text formatting (bold, italic, underline)
-- Intelligent text wrapping and indentation
-- Support for:
-  - Headers with hierarchical styling
-  - Lists (ordered and unordered)
-  - Code blocks (fenced and indented)
-  - Blockquotes
-  - Links with OSC8 hyperlink support
-  - Inline code, bold, and italic
-  - Horizontal rules
-  - Images
-- Terminal width detection and adaptive wrapping
-- Configurable margins, padding, and styling options
-
-## Origin
-
-This library is a C++ port of the terminal renderer from [lowdown](https://github.com/kristapsdz/lowdown) by Kristaps Dzonsons, adapted to work with the [cmark](https://github.com/commonmark/cmark) CommonMark implementation.
-
-## License
-
-ISC License (same as lowdown)
-
-## Dependencies
-
-- cmark >= 0.31.0
-- C++20 compiler

src/libcmarkcpp/include/cmark/cmark-cpp.hh

@@ -1,89 +0,0 @@
-#pragma once
-/**
- * @file cmark-cpp.hh
- * @brief C++ wrappers for the cmark CommonMark library
- */
-
-#include <cmark.h>
-#include <memory>
-#include <string_view>
-#include <cassert>
-
-namespace cmark {
-
-using Node = struct cmark_node;
-using NodeType = cmark_node_type;
-using ListType = cmark_list_type;
-
-using Iter = struct cmark_iter;
-
-struct Deleter
-{
-    void operator()(Node * ptr)
-    {
-        cmark_node_free(ptr);
-    }
-
-    void operator()(Iter * ptr)
-    {
-        cmark_iter_free(ptr);
-    }
-
-    void operator()(char * ptr)
-    {
-        free(ptr);
-    }
-};
-
-template<typename T>
-using UniquePtr = std::unique_ptr<Node, Deleter>;
-
-static inline void parse_document(Node & root, std::string_view s, int options)
-{
-    cmark_parser * parser = cmark_parser_new_with_mem_into_root(options, cmark_get_default_mem_allocator(), &root);
-    cmark_parser_feed(parser, s.data(), s.size());
-    (void) cmark_parser_finish(parser);
-    cmark_parser_free(parser);
-}
-
-static inline UniquePtr<Node> parse_document(std::string_view s, int options)
-{
-    return UniquePtr<Node>{cmark_parse_document(s.data(), s.size(), options)};
-}
-
-static inline std::unique_ptr<char, Deleter> render_commonmark(Node & root, int options, int width)
-{
-    return std::unique_ptr<char, Deleter>{cmark_render_commonmark(&root, options, width)};
-}
-
-static inline std::unique_ptr<char, Deleter> render_xml(Node & root, int options)
-{
-    return std::unique_ptr<char, Deleter>{cmark_render_xml(&root, options)};
-}
-
-static inline UniquePtr<Node> node_new(NodeType type)
-{
-    return UniquePtr<Node>{cmark_node_new(type)};
-}
-
-/**
- * The parent takes ownership
- */
-static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
-{
-    auto status = (bool) cmark_node_append_child(&node, &*child);
-    assert(status);
-    return *child.release();
-}
-
-static inline bool node_set_literal(Node & node, const char * content)
-{
-    return (bool) cmark_node_set_literal(&node, content);
-}
-
-static inline bool node_set_list_type(Node & node, ListType type)
-{
-    return (bool) cmark_node_set_list_type(&node, type);
-}
-
-} // namespace cmark

src/libcmarkcpp/include/cmark/cmark-terminal.hh

@@ -1,108 +0,0 @@
-#pragma once
-///@file
-
-#include "cmark-cpp.hh"
-
-#include <string>
-#include <vector>
-#include <memory>
-
-namespace cmark {
-
-/**
- * Terminal rendering options
- */
-struct TerminalOptions
-{
-    /** Terminal width in columns */
-    size_t cols = 80;
-
-    /** Content width (0 = auto, max 80 or cols) */
-    size_t width = 0;
-
-    /** Horizontal margin (left padding) */
-    size_t hmargin = 0;
-
-    /** Horizontal padding (additional left padding) */
-    size_t hpadding = 4;
-
-    /** Vertical margin (blank lines before/after) */
-    size_t vmargin = 0;
-
-    /** Center content */
-    bool centre = false;
-
-    /** Disable ANSI escape sequences */
-    bool noAnsi = false;
-
-    /** Disable ANSI colors only */
-    bool noColor = false;
-
-    /** Don't show any link URLs */
-    bool noLink = false;
-
-    /** Don't show relative link URLs */
-    bool noRelLink = false;
-
-    /** Shorten long URLs */
-    bool shortLink = false;
-};
-
-/**
- * Style attributes for terminal output
- */
-struct Style
-{
-    bool italic = false;
-    bool strike = false;
-    bool bold = false;
-    bool under = false;
-    size_t bcolour = 0; // Background color (ANSI code)
-    size_t colour = 0;  // Foreground color (ANSI code)
-    int override = 0;   // Override flags
-
-    static constexpr int OVERRIDE_UNDER = 0x01;
-    static constexpr int OVERRIDE_BOLD = 0x02;
-
-    bool hasStyle() const
-    {
-        return colour || bold || italic || under || strike || bcolour || override;
-    }
-};
-
-/**
- * Terminal renderer for CommonMark documents
- *
- * Renders a CMark AST to ANSI terminal output with styling, wrapping,
- * and proper indentation.
- */
-class TerminalRenderer
-{
-public:
-    /**
-     * Create a new terminal renderer with the given options
-     */
-    explicit TerminalRenderer(const TerminalOptions & opts = TerminalOptions{});
-
-    ~TerminalRenderer();
-
-    // Non-copyable
-    TerminalRenderer(const TerminalRenderer &) = delete;
-    TerminalRenderer & operator=(const TerminalRenderer &) = delete;
-
-    /**
-     * Render a CMark node tree to a string
-     */
-    std::string render(cmark::Node & root);
-
-private:
-    struct Impl;
-    std::unique_ptr<Impl> impl;
-};
-
-/**
- * Convenience function to render a CMark document to terminal output
- */
-std::string renderTerminal(cmark::Node & root, const TerminalOptions & opts = TerminalOptions{});
-
-} // namespace cmark

src/libcmarkcpp/meson.build

@@ -1,57 +0,0 @@
-project(
-  'cmark-cpp',
-  'cpp',
-  version : '0.1',
-  default_options : [
-    'cpp_std=c++20',
-    'warning_level=1',
-  ],
-  meson_version : '>= 1.1',
-  license : 'ISC',
-)
-
-cxx = meson.get_compiler('cpp')
-
-# CMark dependency
-cmark = dependency('libcmark', required : true)
-
-sources = files(
-  'cmark-terminal.cc',
-)
-
-headers = files(
-  'include/cmark/cmark-cpp.hh',
-  'include/cmark/cmark-terminal.hh',
-)
-
-include_dirs = include_directories('include')
-
-libcmarkcpp = library(
-  'cmarkcpp',
-  sources,
-  dependencies : cmark,
-  include_directories : include_dirs,
-  install : true,
-  version : meson.project_version(),
-)
-
-install_headers(headers, subdir : 'cmark')
-
-# Make this available as a dependency for meson projects
-meson.override_dependency(
-  'cmark-cpp',
-  declare_dependency(
-    include_directories : include_dirs,
-    link_with : libcmarkcpp,
-    dependencies : cmark,
-  ),
-)
-
-# Pkg-config file
-pkg = import('pkgconfig')
-pkg.generate(
-  libcmarkcpp,
-  name : 'cmark-cpp',
-  description : 'C++ terminal renderer for CommonMark',
-  url : 'https://github.com/NixOS/nix',
-)

src/libcmd/common-eval-args.cc

@@ -34,7 +34,7 @@ EvalSettings evalSettings{
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
                 auto [accessor, lockedRef] =
-                    flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
+                    flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
                 auto storePath = nix::fetchToStore(
                     state.fetchSettings,
                     *state.store,
@@ -180,7 +180,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
 SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir)
 {
     if (EvalSettings::isPseudoUrl(s)) {
-        auto accessor = fetchers::downloadTarball(*state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
+        auto accessor = fetchers::downloadTarball(state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s));
         auto storePath = fetchToStore(state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy);
         return state.storePath(storePath);
     }
@@ -188,8 +188,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
-        auto [accessor, lockedRef] =
-            flakeRef.resolve(fetchSettings, *state.store).lazyFetch(fetchSettings, *state.store);
+        auto [accessor, lockedRef] = flakeRef.resolve(fetchSettings, state.store).lazyFetch(fetchSettings, state.store);
         auto storePath = nix::fetchToStore(
             state.fetchSettings, *state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName());
         state.allowPath(storePath);

src/libcmd/installables.cc

@@ -410,7 +410,7 @@ void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::strin
     Args::completeDir(completions, 0, prefix);
 
     /* Look for registry entries that match the prefix. */
-    for (auto & registry : fetchers::getRegistries(fetchSettings, *store)) {
+    for (auto & registry : fetchers::getRegistries(fetchSettings, store)) {
         for (auto & entry : registry->entries) {
             auto from = entry.from.to_string();
             if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {

src/libcmd/markdown.cc

@@ -1,38 +1,79 @@
 #include "nix/cmd/markdown.hh"
 #include "nix/util/environment-variables.hh"
 #include "nix/util/error.hh"
+#include "nix/util/finally.hh"
 #include "nix/util/terminal.hh"
 
-#include <cmark/cmark-cpp.hh>
-#include <cmark/cmark-terminal.hh>
+#include "cmd-config-private.hh"
+
+#if HAVE_LOWDOWN
+#  include <sys/queue.h>
+#  include <lowdown.h>
+#endif
 
 namespace nix {
 
+#if HAVE_LOWDOWN
 static std::string doRenderMarkdownToTerminal(std::string_view markdown)
 {
     int windowWidth = getWindowSize().second;
 
-    // Set up terminal rendering options
-    ::cmark::TerminalOptions opts;
-    opts.cols = std::max(windowWidth - 5, 60);
-    opts.hmargin = 0;
-    opts.vmargin = 0;
-    opts.noRelLink = true; // Skip rendering relative links
+#  if HAVE_LOWDOWN_1_4
+    struct lowdown_opts_term opts_term{
+        .cols = (size_t) std::max(windowWidth - 5, 60),
+        .hmargin = 0,
+        .vmargin = 0,
+    };
+#  endif
+    struct lowdown_opts opts{
+        .type = LOWDOWN_TERM,
+#  if HAVE_LOWDOWN_1_4
+        .term = opts_term,
+#  endif
+        .maxdepth = 20,
+#  if !HAVE_LOWDOWN_1_4
+        .cols = (size_t) std::max(windowWidth - 5, 60),
+        .hmargin = 0,
+        .vmargin = 0,
+#  endif
+        .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
+        .oflags =
+#  if HAVE_LOWDOWN_1_4
+            LOWDOWN_TERM_NORELLINK // To render full links while skipping relative ones
+#  else
+            LOWDOWN_TERM_NOLINK
+#  endif
+    };
 
     if (!isTTY())
-        opts.noAnsi = true;
+        opts.oflags |= LOWDOWN_TERM_NOANSI;
 
-    // Parse the markdown document
-    auto doc = ::cmark::parse_document(markdown, CMARK_OPT_DEFAULT);
+    auto doc = lowdown_doc_new(&opts);
     if (!doc)
-        throw Error("cannot parse Markdown document");
+        throw Error("cannot allocate Markdown document");
+    Finally freeDoc([&]() { lowdown_doc_free(doc); });
 
-    try {
-        // Render to terminal
-        return ::cmark::renderTerminal(*doc, opts);
-    } catch (const std::exception & e) {
-        throw Error("error rendering Markdown: %s", e.what());
-    }
+    size_t maxn = 0;
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr);
+    if (!node)
+        throw Error("cannot parse Markdown document");
+    Finally freeNode([&]() { lowdown_node_free(node); });
+
+    auto renderer = lowdown_term_new(&opts);
+    if (!renderer)
+        throw Error("cannot allocate Markdown renderer");
+    Finally freeRenderer([&]() { lowdown_term_free(renderer); });
+
+    auto buf = lowdown_buf_new(16384);
+    if (!buf)
+        throw Error("cannot allocate Markdown output buffer");
+    Finally freeBuffer([&]() { lowdown_buf_free(buf); });
+
+    int rndr_res = lowdown_term_rndr(buf, renderer, node);
+    if (!rndr_res)
+        throw Error("allocation error while rendering Markdown");
+
+    return std::string(buf->data, buf->size);
 }
 
 std::string renderMarkdownToTerminal(std::string_view markdown)
@@ -43,4 +84,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         return doRenderMarkdownToTerminal(markdown);
 }
 
+#else
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    return std::string(markdown);
+}
+#endif
+
 } // namespace nix

src/libcmd/meson.build

@@ -26,13 +26,25 @@ deps_public_maybe_subproject = [
   dependency('nix-expr'),
   dependency('nix-flake'),
   dependency('nix-main'),
-  dependency('cmark-cpp'),
 ]
 subdir('nix-meson-build-support/subprojects')
 
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
+lowdown = dependency(
+  'lowdown',
+  version : '>= 0.9.0',
+  required : get_option('markdown'),
+)
+deps_private += lowdown
+configdata.set('HAVE_LOWDOWN', lowdown.found().to_int())
+# The API changed slightly around terminal initialization.
+configdata.set(
+  'HAVE_LOWDOWN_1_4',
+  lowdown.version().version_compare('>= 1.4.0').to_int(),
+)
+
 readline_flavor = get_option('readline-flavor')
 if readline_flavor == 'editline'
   editline = dependency('libeditline', 'editline', version : '>=1.14')

src/libcmd/meson.options

@@ -1,5 +1,11 @@
 # vim: filetype=meson
 
+option(
+  'markdown',
+  type : 'feature',
+  description : 'Enable Markdown rendering in the Nix binary (requires lowdown)',
+)
+
 option(
   'readline-flavor',
   type : 'combo',

src/libcmd/package.nix

@@ -11,12 +11,16 @@
   nix-main,
   editline,
   readline,
+  lowdown,
   nlohmann_json,
 
   # Configuration Options
 
   version,
 
+  # Whether to enable Markdown rendering in the Nix binary.
+  enableMarkdown ? !stdenv.hostPlatform.isWindows,
+
   # Which interactive line editor library to use for Nix's repl.
   #
   # Currently supported choices are:
@@ -49,7 +53,8 @@ mkMesonLibrary (finalAttrs: {
 
   buildInputs = [
     ({ inherit editline readline; }.${readlineFlavor})
-  ];
+  ]
+  ++ lib.optional enableMarkdown lowdown;
 
   propagatedBuildInputs = [
     nix-util
@@ -62,6 +67,7 @@ mkMesonLibrary (finalAttrs: {
   ];
 
   mesonFlags = [
+    (lib.mesonEnable "markdown" enableMarkdown)
     (lib.mesonOption "readline-flavor" readlineFlavor)
   ];
 

src/libcmd/repl.cc

@@ -656,7 +656,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
                             + "\n\n";
             }
 
-            markdown += stripIndentation(doc->doc);
+            markdown += stripIndentation(doc->doc.view());
 
             logger->cout(trim(renderMarkdownToTerminal(markdown)));
         } else if (fallbackPos) {
@@ -739,7 +739,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS)
 
     auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true);
     if (evalSettings.pureEval && !flakeRef.input.isLocked(fetchSettings))
-        throw Error("cannot use ':load-flake' on unlocked flake reference '%s' (use --impure to override)", flakeRefS);
+        throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS);
 
     Value v;
 

src/libexpr-c/nix_api_value.cc

@@ -134,7 +134,7 @@ PrimOp * nix_alloc_primop(
                     .name = name,
                     .args = {},
                     .arity = (size_t) arity,
-                    .doc = doc,
+                    .doc = &nix::StringData::make(doc),
                     .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
         if (args)
             for (size_t i = 0; args[i]; i++)

src/libexpr-tests/json.cc

@@ -1,6 +1,6 @@
 #include "nix/expr/tests/libexpr.hh"
 #include "nix/expr/value-to-json.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 namespace nix {
 // Testing the conversion to JSON

src/libexpr-tests/value/print.cc

@@ -1,5 +1,5 @@
 #include "nix/expr/tests/libexpr.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include "nix/expr/value.hh"
 #include "nix/expr/print.hh"

src/libexpr-tests/value/value.cc

@@ -1,5 +1,5 @@
 #include "nix/expr/value.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include "nix/store/tests/libstore.hh"
 #include <gtest/gtest.h>

src/libexpr/eval.cc

@@ -51,36 +51,6 @@ using json = nlohmann::json;
 
 namespace nix {
 
-/**
- * Just for doc strings. Not for regular string values.
- */
-static char * allocString(size_t size)
-{
-    char * t;
-    t = (char *) GC_MALLOC_ATOMIC(size);
-    if (!t)
-        throw std::bad_alloc();
-    return t;
-}
-
-// When there's no need to write to the string, we can optimize away empty
-// string allocations.
-// This function handles makeImmutableString(std::string_view()) by returning
-// the empty string.
-/**
- * Just for doc strings. Not for regular string values.
- */
-static const char * makeImmutableString(std::string_view s)
-{
-    const size_t size = s.size();
-    if (size == 0)
-        return "";
-    auto t = allocString(size + 1);
-    memcpy(t, s.data(), size);
-    t[size] = '\0';
-    return t;
-}
-
 StringData & StringData::alloc(size_t size)
 {
     void * t = GC_MALLOC_ATOMIC(sizeof(StringData) + size + 1);
@@ -205,7 +175,7 @@ bool Value::isTrivial() const
 {
     return !isa<tApp, tPrimOpApp>()
            && (!isa<tThunk>()
-               || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs->empty())
+               || (dynamic_cast<ExprAttrs *>(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty())
                || dynamic_cast<ExprLambda *>(thunk().expr) || dynamic_cast<ExprList *>(thunk().expr));
 }
 
@@ -517,16 +487,15 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
     if (primOp.arity == 0) {
         primOp.arity = 1;
         auto vPrimOp = allocValue();
-        vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
+        vPrimOp->mkPrimOp(new PrimOp(primOp));
         Value v;
         v.mkApp(vPrimOp, vPrimOp);
-        auto & primOp1 = *vPrimOp->primOp();
         return addConstant(
-            primOp1.name,
+            primOp.name,
             v,
             {
                 .type = nThunk, // FIXME
-                .doc = primOp1.doc ? primOp1.doc->c_str() : nullptr,
+                .doc = primOp.doc,
             });
     }
 
@@ -566,14 +535,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
 {
     if (v.isPrimOp()) {
         auto v2 = &v;
-        auto & primOp = *v2->primOp();
-        if (primOp.doc)
+        if (auto * doc = v2->primOp()->doc)
             return Doc{
                 .pos = {},
-                .name = primOp.name,
-                .arity = primOp.arity,
-                .args = primOp.args,
-                .doc = primOp.doc->c_str(),
+                .name = v2->primOp()->name,
+                .arity = v2->primOp()->arity,
+                .args = v2->primOp()->args,
+                .doc = *doc,
             };
     }
     if (v.isLambda()) {
@@ -615,9 +583,8 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
             .name = name,
             .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though...
             .args = {},
-            /* N.B. Can't use StringData here, because that would lead to an interior pointer.
-               NOTE: memory leak when compiled without GC. */
-            .doc = makeImmutableString(s.view()),
+            /* NOTE: memory leak when compiled without GC. */
+            .doc = StringData::make(s.view()),
         };
     }
     if (isFunctor(v)) {
@@ -1226,26 +1193,26 @@ Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up)
 
 void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 {
-    auto bindings = state.buildBindings(attrs->size() + dynamicAttrs->size());
+    auto bindings = state.buildBindings(attrs.size() + dynamicAttrs.size());
     auto dynamicEnv = &env;
     bool sort = false;
 
     if (recursive) {
         /* Create a new environment that contains the attributes in
            this `rec'. */
-        Env & env2(state.mem.allocEnv(attrs->size()));
+        Env & env2(state.mem.allocEnv(attrs.size()));
         env2.up = &env;
         dynamicEnv = &env2;
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr;
 
-        AttrDefs::iterator overrides = attrs->find(state.s.overrides);
-        bool hasOverrides = overrides != attrs->end();
+        AttrDefs::iterator overrides = attrs.find(state.s.overrides);
+        bool hasOverrides = overrides != attrs.end();
 
         /* The recursive attributes are evaluated in the new
            environment, while the inherited attributes are evaluated
            in the original environment. */
         Displacement displ = 0;
-        for (auto & i : *attrs) {
+        for (auto & i : attrs) {
             Value * vAttr;
             if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) {
                 vAttr = state.allocValue();
@@ -1272,8 +1239,8 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
                 "while evaluating the `__overrides` attribute");
             bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size()));
             for (auto & i : *vOverrides->attrs()) {
-                AttrDefs::iterator j = attrs->find(i.name);
-                if (j != attrs->end()) {
+                AttrDefs::iterator j = attrs.find(i.name);
+                if (j != attrs.end()) {
                     (*bindings.bindings)[j->second.displ] = i;
                     env2.values[j->second.displ] = i.value;
                 } else
@@ -1285,13 +1252,13 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
 
     else {
         Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr;
-        for (auto & i : *attrs)
+        for (auto & i : attrs)
             bindings.insert(
                 i.first, i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), i.second.pos);
     }
 
     /* Dynamic attrs apply *after* rec and __overrides. */
-    for (auto & i : *dynamicAttrs) {
+    for (auto & i : dynamicAttrs) {
         Value nameVal;
         i.nameExpr->eval(state, *dynamicEnv, nameVal);
         state.forceValue(nameVal, i.pos);
@@ -1325,7 +1292,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
 {
     /* Create a new environment that contains the attributes in this
        `let'. */
-    Env & env2(state.mem.allocEnv(attrs->attrs->size()));
+    Env & env2(state.mem.allocEnv(attrs->attrs.size()));
     env2.up = &env;
 
     Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr;
@@ -1334,7 +1301,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v)
        while the inherited attributes are evaluated in the original
        environment. */
     Displacement displ = 0;
-    for (auto & i : *attrs->attrs) {
+    for (auto & i : attrs->attrs) {
         env2.values[displ++] = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv));
     }
 
@@ -3190,7 +3157,7 @@ std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Pat
 
     if (EvalSettings::isPseudoUrl(value)) {
         try {
-            auto accessor = fetchers::downloadTarball(*store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
+            auto accessor = fetchers::downloadTarball(store, fetchSettings, EvalSettings::resolvePseudoUrl(value));
             auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy);
             return finish(this->storePath(storePath));
         } catch (Error & e) {

src/libexpr/include/nix/expr/eval.hh

@@ -108,7 +108,7 @@ struct PrimOp
     /**
      * Optional free-form documentation about the primop.
      */
-    std::optional<std::string> doc;
+    const StringData * doc = nullptr;
 
     /**
      * Add a trace item, while calling the `<name>` builtin.
@@ -156,7 +156,7 @@ struct Constant
     /**
      * Optional free-form documentation about the constant.
      */
-    const char * doc = nullptr;
+    const StringData * doc = nullptr;
 
     /**
      * Whether the constant is impure, and not available in pure mode.
@@ -837,11 +837,7 @@ public:
         std::optional<std::string> name;
         size_t arity;
         std::vector<std::string> args;
-        /**
-         * Unlike the other `doc` fields in this file, this one should never be
-         * `null`.
-         */
-        const char * doc;
+        const StringData & doc;
     };
 
     /**

src/libexpr/include/nix/expr/meson.build

@@ -31,7 +31,8 @@ headers = [ config_pub_h ] + files(
   'print.hh',
   'repl-exit-status.hh',
   'search-path.hh',
-  'static-string-data.hh',
+  'string-data.hh',
+  'string-data-static.hh',
   'symbol-table.hh',
   'value-to-json.hh',
   'value-to-xml.hh',

src/libexpr/include/nix/expr/nixexpr.hh

@@ -12,7 +12,7 @@
 #include "nix/expr/value.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/eval-error.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/util/pos-idx.hh"
 #include "nix/expr/counter.hh"
 #include "nix/util/pos-table.hh"
@@ -395,13 +395,9 @@ struct ExprAttrs : Expr
         }
     };
 
-    typedef std::pmr::map<Symbol, AttrDef> AttrDefs;
-    /**
-     * attrs will never be null. we use std::optional so that we can call emplace() to re-initialize the value with a
-     * new pmr::map using a different allocator (move assignment will copy into the old allocator)
-     */
-    std::optional<AttrDefs> attrs;
-    std::unique_ptr<std::pmr::vector<Expr *>> inheritFromExprs;
+    typedef std::map<Symbol, AttrDef> AttrDefs;
+    AttrDefs attrs;
+    std::unique_ptr<std::vector<Expr *>> inheritFromExprs;
 
     struct DynamicAttrDef
     {
@@ -413,20 +409,13 @@ struct ExprAttrs : Expr
             , pos(pos) {};
     };
 
-    typedef std::pmr::vector<DynamicAttrDef> DynamicAttrDefs;
-    /**
-     * dynamicAttrs will never be null. See comment on AttrDefs above.
-     */
-    std::optional<DynamicAttrDefs> dynamicAttrs;
+    typedef std::vector<DynamicAttrDef> DynamicAttrDefs;
+    DynamicAttrDefs dynamicAttrs;
     ExprAttrs(const PosIdx & pos)
         : recursive(false)
-        , pos(pos)
-        , attrs(AttrDefs{})
-        , dynamicAttrs(DynamicAttrDefs{}) {};
+        , pos(pos) {};
     ExprAttrs()
-        : recursive(false)
-        , attrs(AttrDefs{})
-        , dynamicAttrs(DynamicAttrDefs{}) {};
+        : recursive(false) {};
 
     PosIdx getPos() const override
     {

src/libexpr/include/nix/expr/parser-state.hh

@@ -5,7 +5,7 @@
 
 #include "nix/expr/eval.hh"
 #include "nix/expr/value.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 namespace nix {
 
@@ -126,8 +126,8 @@ inline void ParserState::addAttr(
     for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) {
         ExprAttrs * nested;
         if (i->symbol) {
-            ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(i->symbol);
-            if (j != attrs->attrs->end()) {
+            ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol);
+            if (j != attrs->attrs.end()) {
                 nested = dynamic_cast<ExprAttrs *>(j->second.e);
                 if (!nested) {
                     attrPath.erase(i + 1, attrPath.end());
@@ -135,11 +135,11 @@ inline void ParserState::addAttr(
                 }
             } else {
                 nested = exprs.add<ExprAttrs>();
-                (*attrs->attrs)[i->symbol] = ExprAttrs::AttrDef(nested, pos);
+                attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos);
             }
         } else {
             nested = exprs.add<ExprAttrs>();
-            attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
+            attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos));
         }
         attrs = nested;
     }
@@ -148,7 +148,7 @@ inline void ParserState::addAttr(
     if (i->symbol) {
         addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos));
     } else {
-        attrs->dynamicAttrs->push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
+        attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos));
     }
 
     auto it = lexerState.positionToDocComment.find(pos);
@@ -165,8 +165,8 @@ inline void ParserState::addAttr(
 inline void
 ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def)
 {
-    ExprAttrs::AttrDefs::iterator j = attrs->attrs->find(symbol);
-    if (j != attrs->attrs->end()) {
+    ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol);
+    if (j != attrs->attrs.end()) {
         // This attr path is already defined. However, if both
         // e and the expr pointed by the attr path are two attribute sets,
         // we want to merge them.
@@ -181,8 +181,8 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
         // See https://github.com/NixOS/nix/issues/9020.
         if (jAttrs && ae) {
             if (ae->inheritFromExprs && !jAttrs->inheritFromExprs)
-                jAttrs->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>();
-            for (auto & ad : *ae->attrs) {
+                jAttrs->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
+            for (auto & ad : ae->attrs) {
                 if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) {
                     auto & sel = dynamic_cast<ExprSelect &>(*ad.second.e);
                     auto & from = dynamic_cast<ExprInheritFrom &>(*sel.e);
@@ -192,12 +192,12 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
                 addAttr(jAttrs, attrPath, ad.first, std::move(ad.second));
                 attrPath.pop_back();
             }
-            ae->attrs->clear();
-            jAttrs->dynamicAttrs->insert(
-                jAttrs->dynamicAttrs->end(),
-                std::make_move_iterator(ae->dynamicAttrs->begin()),
-                std::make_move_iterator(ae->dynamicAttrs->end()));
-            ae->dynamicAttrs->clear();
+            ae->attrs.clear();
+            jAttrs->dynamicAttrs.insert(
+                jAttrs->dynamicAttrs.end(),
+                std::make_move_iterator(ae->dynamicAttrs.begin()),
+                std::make_move_iterator(ae->dynamicAttrs.end()));
+            ae->dynamicAttrs.clear();
             if (ae->inheritFromExprs) {
                 jAttrs->inheritFromExprs->insert(
                     jAttrs->inheritFromExprs->end(),
@@ -210,7 +210,7 @@ ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symb
         }
     } else {
         // This attr path is not defined. Let's create it.
-        attrs->attrs->emplace(symbol, def);
+        attrs->attrs.emplace(symbol, def);
         def.e->setName(symbol);
     }
 }

src/libexpr/include/nix/expr/string-data-static.hh

@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "nix/expr/value.hh"
+#include "nix/expr/string-data.hh"
 
 namespace nix {
 

src/libexpr/include/nix/expr/string-data.hh

@@ -0,0 +1,96 @@
+#pragma once
+///@file
+
+#include <cstddef>
+#include <cstring>
+#include <memory_resource>
+#include <string_view>
+
+namespace nix {
+
+class StringData
+{
+public:
+    using size_type = std::size_t;
+
+    size_type size_;
+    char data_[];
+
+    /*
+     * This in particular ensures that we cannot have a `StringData`
+     * that we use by value, which is just what we want!
+     *
+     * Dynamically sized types aren't a thing in C++ and even flexible array
+     * members are a language extension and beyond the realm of standard C++.
+     * Technically, sizeof data_ member is 0 and the intended way to use flexible
+     * array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
+     * and the compiler will consider alignment restrictions for the FAM.
+     *
+     */
+
+    StringData(StringData &&) = delete;
+    StringData & operator=(StringData &&) = delete;
+    StringData(const StringData &) = delete;
+    StringData & operator=(const StringData &) = delete;
+    ~StringData() = default;
+
+private:
+    StringData() = delete;
+
+    explicit StringData(size_type size)
+        : size_(size)
+    {
+    }
+
+public:
+    /**
+     * Allocate StringData on the (possibly) GC-managed heap and copy
+     * the contents of s to it.
+     */
+    static const StringData & make(std::string_view s);
+
+    /**
+     * Allocate StringData on the (possibly) GC-managed heap.
+     * @param size Length of the string (without the NUL terminator).
+     */
+    static StringData & alloc(size_t size);
+
+    size_t size() const
+    {
+        return size_;
+    }
+
+    char * data() noexcept
+    {
+        return data_;
+    }
+
+    const char * data() const noexcept
+    {
+        return data_;
+    }
+
+    const char * c_str() const noexcept
+    {
+        return data_;
+    }
+
+    constexpr std::string_view view() const noexcept
+    {
+        return std::string_view(data_, size_);
+    }
+
+    template<size_t N>
+    struct Static;
+
+    static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
+    {
+        auto & res =
+            *new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
+        std::memcpy(res.data_, s.data(), s.size());
+        res.data_[s.size()] = '\0';
+        return res;
+    }
+};
+
+} // namespace nix

src/libexpr/include/nix/expr/symbol-table.hh

@@ -3,7 +3,7 @@
 
 #include <memory_resource>
 #include "nix/expr/value.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/util/chunked-vector.hh"
 #include "nix/util/error.hh"
 

src/libexpr/include/nix/expr/value.hh

@@ -3,16 +3,14 @@
 
 #include <bit>
 #include <cassert>
-#include <cstddef>
-#include <cstring>
 #include <memory>
-#include <memory_resource>
 #include <span>
 #include <string_view>
 #include <type_traits>
 #include <concepts>
 
 #include "nix/expr/eval-gc.hh"
+#include "nix/expr/string-data.hh"
 #include "nix/expr/value/context.hh"
 #include "nix/util/source-path.hh"
 #include "nix/expr/print-options.hh"
@@ -193,91 +191,6 @@ public:
     friend struct Value;
 };
 
-class StringData
-{
-public:
-    using size_type = std::size_t;
-
-    size_type size_;
-    char data_[];
-
-    /*
-     * This in particular ensures that we cannot have a `StringData`
-     * that we use by value, which is just what we want!
-     *
-     * Dynamically sized types aren't a thing in C++ and even flexible array
-     * members are a language extension and beyond the realm of standard C++.
-     * Technically, sizeof data_ member is 0 and the intended way to use flexible
-     * array members is to allocate sizeof(StrindData) + count * sizeof(char) bytes
-     * and the compiler will consider alignment restrictions for the FAM.
-     *
-     */
-
-    StringData(StringData &&) = delete;
-    StringData & operator=(StringData &&) = delete;
-    StringData(const StringData &) = delete;
-    StringData & operator=(const StringData &) = delete;
-    ~StringData() = default;
-
-private:
-    StringData() = delete;
-
-    explicit StringData(size_type size)
-        : size_(size)
-    {
-    }
-
-public:
-    /**
-     * Allocate StringData on the (possibly) GC-managed heap and copy
-     * the contents of s to it.
-     */
-    static const StringData & make(std::string_view s);
-
-    /**
-     * Allocate StringData on the (possibly) GC-managed heap.
-     * @param size Length of the string (without the NUL terminator).
-     */
-    static StringData & alloc(size_t size);
-
-    size_t size() const
-    {
-        return size_;
-    }
-
-    char * data() noexcept
-    {
-        return data_;
-    }
-
-    const char * data() const noexcept
-    {
-        return data_;
-    }
-
-    const char * c_str() const noexcept
-    {
-        return data_;
-    }
-
-    constexpr std::string_view view() const noexcept
-    {
-        return std::string_view(data_, size_);
-    }
-
-    template<size_t N>
-    struct Static;
-
-    static StringData & make(std::pmr::memory_resource & resource, std::string_view s)
-    {
-        auto & res =
-            *new (resource.allocate(sizeof(StringData) + s.size() + 1, alignof(StringData))) StringData(s.size());
-        std::memcpy(res.data_, s.data(), s.size());
-        res.data_[s.size()] = '\0';
-        return res;
-    }
-};
-
 namespace detail {
 
 /**

src/libexpr/meson.build

@@ -24,7 +24,6 @@ deps_public_maybe_subproject = [
   dependency('nix-util'),
   dependency('nix-store'),
   dependency('nix-fetchers'),
-  dependency('cmark-cpp'),
 ]
 subdir('nix-meson-build-support/subprojects')
 subdir('nix-meson-build-support/big-objs')

src/libexpr/nixexpr.cc

@@ -74,9 +74,9 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const
 {
-    typedef const AttrDefs::value_type * Attr;
+    typedef const decltype(attrs)::value_type * Attr;
     std::vector<Attr> sorted;
-    for (auto & i : *attrs)
+    for (auto & i : attrs)
         sorted.push_back(&i);
     std::sort(sorted.begin(), sorted.end(), [&](Attr a, Attr b) {
         std::string_view sa = symbols[a->first], sb = symbols[b->first];
@@ -122,7 +122,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co
             str << "; ";
         }
     }
-    for (auto & i : *dynamicAttrs) {
+    for (auto & i : dynamicAttrs) {
         str << "\"${";
         i.nameExpr->show(symbols, str);
         str << "}\" = ";
@@ -401,26 +401,15 @@ ExprAttrs::bindInheritSources(EvalState & es, const std::shared_ptr<const Static
 
 void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
-    // Move storage into the Exprs arena
-    {
-        auto arena = es.mem.exprs.alloc;
-        AttrDefs newAttrs{std::move(*attrs), arena};
-        attrs.emplace(std::move(newAttrs), arena);
-        DynamicAttrDefs newDynamicAttrs{std::move(*dynamicAttrs), arena};
-        dynamicAttrs.emplace(std::move(newDynamicAttrs), arena);
-        if (inheritFromExprs)
-            inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>(std::move(*inheritFromExprs), arena);
-    }
-
     if (es.debugRepl)
         es.exprEnvs.insert(std::make_pair(this, env));
 
     if (recursive) {
         auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
-            auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->size());
+            auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs.size());
 
             Displacement displ = 0;
-            for (auto & i : *attrs)
+            for (auto & i : attrs)
                 newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
             return newEnv;
         }();
@@ -428,20 +417,20 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv>
         // No need to sort newEnv since attrs is in sorted order.
 
         auto inheritFromEnv = bindInheritSources(es, newEnv);
-        for (auto & i : *attrs)
+        for (auto & i : attrs)
             i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
-        for (auto & i : *dynamicAttrs) {
+        for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, newEnv);
             i.valueExpr->bindVars(es, newEnv);
         }
     } else {
         auto inheritFromEnv = bindInheritSources(es, env);
 
-        for (auto & i : *attrs)
+        for (auto & i : attrs)
             i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv));
 
-        for (auto & i : *dynamicAttrs) {
+        for (auto & i : dynamicAttrs) {
             i.nameExpr->bindVars(es, env);
             i.valueExpr->bindVars(es, env);
         }
@@ -497,10 +486,10 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
 void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env)
 {
     auto newEnv = [&]() -> std::shared_ptr<const StaticEnv> {
-        auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs->size());
+        auto newEnv = std::make_shared<StaticEnv>(nullptr, env, attrs->attrs.size());
 
         Displacement displ = 0;
-        for (auto & i : *attrs->attrs)
+        for (auto & i : attrs->attrs)
             newEnv->vars.emplace_back(i.first, i.second.displ = displ++);
         return newEnv;
     }();
@@ -508,7 +497,7 @@ void ExprLet::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> &
     // No need to sort newEnv since attrs->attrs is in sorted order.
 
     auto inheritFromEnv = attrs->bindInheritSources(es, newEnv);
-    for (auto & i : *attrs->attrs)
+    for (auto & i : attrs->attrs)
         i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv));
 
     if (es.debugRepl)

src/libexpr/parser.y

@@ -211,7 +211,7 @@ expr_function
   | WITH expr ';' expr_function
     { $$ = state->exprs.add<ExprWith>(CUR_POS, $2, $4); }
   | LET binds IN_KW expr_function
-    { if (!$2->dynamicAttrs->empty())
+    { if (!$2->dynamicAttrs.empty())
         throw ParseError({
             .msg = HintFmt("dynamic attributes not allowed in let"),
             .pos = state->positions[CUR_POS]
@@ -413,9 +413,9 @@ binds1
   | binds[accum] INHERIT attrs ';'
     { $$ = $accum;
       for (auto & [i, iPos] : $attrs) {
-          if ($accum->attrs->find(i.symbol) != $accum->attrs->end())
-              state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos);
-          $accum->attrs->emplace(
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(state->exprs.add<ExprVar>(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited));
       }
@@ -423,13 +423,13 @@ binds1
   | binds[accum] INHERIT '(' expr ')' attrs ';'
     { $$ = $accum;
       if (!$accum->inheritFromExprs)
-          $accum->inheritFromExprs = std::make_unique<std::pmr::vector<Expr *>>();
+          $accum->inheritFromExprs = std::make_unique<std::vector<Expr *>>();
       $accum->inheritFromExprs->push_back($expr);
       auto from = state->exprs.add<ExprInheritFrom>(state->at(@expr), $accum->inheritFromExprs->size() - 1);
       for (auto & [i, iPos] : $attrs) {
-          if ($accum->attrs->find(i.symbol) != $accum->attrs->end())
-              state->dupAttr(i.symbol, iPos, (*$accum->attrs)[i.symbol].pos);
-          $accum->attrs->emplace(
+          if ($accum->attrs.find(i.symbol) != $accum->attrs.end())
+              state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos);
+          $accum->attrs.emplace(
               i.symbol,
               ExprAttrs::AttrDef(
                   state->exprs.add<ExprSelect>(state->exprs.alloc, iPos, from, i.symbol),

src/libexpr/primops/context.cc

@@ -1,5 +1,6 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/store/derivations.hh"
 #include "nix/store/store-api.hh"
 #include "nix/store/globals.hh"
@@ -17,9 +18,9 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos,
 static RegisterPrimOp primop_unsafeDiscardStringContext({
     .name = "__unsafeDiscardStringContext",
     .args = {"s"},
-    .doc = R"(
+    .doc = &R"(
         Discard the [string context](@docroot@/language/string-context.md) from a value that can be coerced to a string.
-    )",
+    )"_sds,
     .fun = prim_unsafeDiscardStringContext,
 });
 
@@ -33,7 +34,7 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value ** args,
 static RegisterPrimOp primop_hasContext(
     {.name = "__hasContext",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Return `true` if string *s* has a non-empty context.
       The context can be obtained with
       [`getContext`](#builtins-getContext).
@@ -50,7 +51,7 @@ static RegisterPrimOp primop_hasContext(
       > then throw "package name cannot contain string context"
       > else { ${name} = meta; }
       > ```
-    )",
+    )"_sds,
      .fun = prim_hasContext});
 
 static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@@ -75,7 +76,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
 static RegisterPrimOp primop_unsafeDiscardOutputDependency(
     {.name = "__unsafeDiscardOutputDependency",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Create a copy of the given string where every
       [derivation deep](@docroot@/language/string-context.md#string-context-element-derivation-deep)
       string context element is turned into a
@@ -91,7 +92,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency(
       Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything.
 
       [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies
-    )",
+    )"_sds,
      .fun = prim_unsafeDiscardOutputDependency});
 
 static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value ** args, Value & v)
@@ -143,7 +144,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V
 static RegisterPrimOp primop_addDrvOutputDependencies(
     {.name = "__addDrvOutputDependencies",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Create a copy of the given string where a single
       [constant](@docroot@/language/string-context.md#string-context-constant)
       string context element is turned into a
@@ -156,7 +157,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies(
       The latter is supported so this function is idempotent.
 
       This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency).
-    )",
+    )"_sds,
      .fun = prim_addDrvOutputDependencies});
 
 /* Extract the context of a string as a structured Nix value.
@@ -230,7 +231,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args,
 static RegisterPrimOp primop_getContext(
     {.name = "__getContext",
      .args = {"s"},
-     .doc = R"(
+     .doc = &R"(
       Return the string context of *s*.
 
       The string context tracks references to derivations within a string.
@@ -248,7 +249,7 @@ static RegisterPrimOp primop_getContext(
       ```
       { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; }
       ```
-    )",
+    )"_sds,
      .fun = prim_getContext});
 
 /* Append the given context to a given string.

src/libexpr/primops/fetchClosure.cc

@@ -1,4 +1,5 @@
 #include "nix/expr/primops.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/store/store-open.hh"
 #include "nix/store/realisation.hh"
 #include "nix/store/make-content-addressed.hh"
@@ -216,7 +217,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args
 static RegisterPrimOp primop_fetchClosure({
     .name = "__fetchClosure",
     .args = {"args"},
-    .doc = R"(
+    .doc = &R"(
       Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context.
 
       This function can be invoked in three ways that we will discuss in order of preference.
@@ -284,7 +285,7 @@ static RegisterPrimOp primop_fetchClosure({
       `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression.
       However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched.
       Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity.
-    )",
+    )"_sds,
     .fun = prim_fetchClosure,
     .experimentalFeature = Xp::FetchClosure,
 });

src/libexpr/primops/fetchMercurial.cc

@@ -81,7 +81,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar
         attrs.insert_or_assign("rev", rev->gitRev());
     auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs));
 
-    auto [storePath, input2] = input.fetchToStore(state.fetchSettings, *state.store);
+    auto [storePath, input2] = input.fetchToStore(state.fetchSettings, state.store);
 
     auto attrs2 = state.buildBindings(8);
     state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath));

src/libexpr/primops/fetchTree.cc

@@ -2,13 +2,13 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
 #include "nix/expr/eval-settings.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/store/store-api.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/store/filetransfer.hh"
 #include "nix/fetchers/registry.hh"
 #include "nix/fetchers/tarball.hh"
 #include "nix/util/url.hh"
-#include "nix/util/cmark-cpp.hh"
 #include "nix/expr/value-to-json.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/fetchers/input-cache.hh"
@@ -195,7 +195,7 @@ static void fetchTree(
     }
 
     if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes))
-        input = lookupInRegistries(state.fetchSettings, *state.store, input, fetchers::UseRegistries::Limited).first;
+        input = lookupInRegistries(state.fetchSettings, state.store, input, fetchers::UseRegistries::Limited).first;
 
     if (state.settings.pureEval && !input.isLocked(state.fetchSettings)) {
         if (input.getNarHash())
@@ -221,7 +221,7 @@ static void fetchTree(
     }
 
     auto cachedInput =
-        state.inputCache->getAccessor(state.fetchSettings, *state.store, input, fetchers::UseRegistries::No);
+        state.inputCache->getAccessor(state.fetchSettings, state.store, input, fetchers::UseRegistries::No);
 
     auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor);
 
@@ -236,144 +236,229 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, V
 static RegisterPrimOp primop_fetchTree({
     .name = "fetchTree",
     .args = {"input"},
-    .doc = []() -> std::string {
-        using namespace cmark;
+    .doc = &R"(
+      Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
 
-        // Stores strings referenced by AST. Deallocate after rendering.
-        std::vector<std::string> textArena;
+      - the resulting fixed-output [store path](@docroot@/store/store-path.md)
+      - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
+      - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
 
-        auto root = node_new(CMARK_NODE_DOCUMENT);
+      *input* must be an attribute set with the following attributes:
 
-        auto & before = textArena.emplace_back(stripIndentation(R"(
-          Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
+      - `type` (String, required)
 
-          - the resulting fixed-output [store path](@docroot@/store/store-path.md)
-          - the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
-          - backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
+        One of the [supported source types](#source-types).
+        This determines other required and allowed input attributes.
 
-          *input* must be an attribute set with the following attributes:
+      - `narHash` (String, optional)
 
-          - `type` (String, required)
+        The `narHash` parameter can be used to substitute the source of the tree.
+        It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
+        If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
 
-            One of the [supported source types](#source-types).
-            This determines other required and allowed input attributes.
+      A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
+      That is, `fetchTree` is idempotent.
 
-          - `narHash` (String, optional)
+      Downloads are cached in `$XDG_CACHE_HOME/nix`.
+      The remote source is fetched from the network if both are true:
+      - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
 
-            The `narHash` parameter can be used to substitute the source of the tree.
-            It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
-            If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
+        > **Note**
+        >
+        > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
 
-          A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
-          That is, `fetchTree` is idempotent.
+      - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
 
-          Downloads are cached in `$XDG_CACHE_HOME/nix`.
-          The remote source is fetched from the network if both are true:
-          - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
+      ## Source types
 
-            > **Note**
+      The following source types and associated input attributes are supported.
+
+      <!-- TODO: It would be soooo much more predictable to work with (and
+      document) if `fetchTree` was a curried call with the first parameter for
+      `type` or an attribute like `builtins.fetchTree.git`! -->
+
+      - `"file"`
+
+        Place a plain file into the Nix store.
+        This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
+
+        - `url` (String, required)
+
+          Supported protocols:
+
+          - `https`
+
+            > **Example**
             >
-            > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "https://example.com/index.html";
+            > }
+            > ```
 
-          - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
+          - `http`
 
-          ## Source types
+            Insecure HTTP transfer for legacy sources.
 
-          The following source types and associated input attributes are supported.
+            > **Warning**
+            >
+            > HTTP performs no encryption or authentication.
+            > Use a `narHash` known in advance to ensure the output has expected contents.
 
-          <!-- TODO: It would be soooo much more predictable to work with (and
-          document) if `fetchTree` was a curried call with the first parameter for
-          `type` or an attribute like `builtins.fetchTree.git`! -->
-        )"));
-        parse_document(*root, before, CMARK_OPT_DEFAULT);
+          - `file`
 
-        auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));
+            A file on the local file system.
 
-        for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
-            auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
-            {
-                auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
-                auto & name = node_append_child(name_p, node_new(CMARK_NODE_CODE));
-                auto & name_t = textArena.emplace_back(quoteString(schemeName, '"'));
-                node_set_literal(name, name_t.c_str());
-            }
-            parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT);
+            > **Example**
+            >
+            > ```nix
+            > fetchTree {
+            >   type = "file";
+            >   url = "file:///home/eelco/nix/README.md";
+            > }
+            > ```
 
-            auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
-            for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
-                auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
-                {
-                    auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
-                    {
-                        auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE));
-                        auto & name_t = textArena.emplace_back(attrName);
-                        node_set_literal(name, name_t.c_str());
-                    }
-                    auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT));
-                    auto & header = textArena.emplace_back(
-                        std::string{} + " (" + attribute.type + ", " + (attribute.required ? "required" : "optional")
-                        + ")");
-                    node_set_literal(info, header.c_str());
-                }
-                {
-                    auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
-                    parse_document(a, doc, CMARK_OPT_DEFAULT);
-                }
-            }
-        }
+      - `"git"`
 
-        auto & after = textArena.emplace_back(stripIndentation(R"(
-          The following input types are still subject to change:
+        Fetch a Git tree and copy it to the Nix store.
+        This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
 
-          - `"path"`
-          - `"github"`
-          - `"gitlab"`
-          - `"sourcehut"`
-          - `"mercurial"`
+        - `allRefs` (Bool, optional)
 
-         *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
-         The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          Whether to fetch all references (eg. branches and tags) of the repository.
+          With this argument being true, it's possible to load a `rev` from *any* `ref`.
+          (Without setting this option, only `rev`s from the specified `ref` are supported).
+
+          Default: `false`
+
+        - `lastModified` (Integer, optional)
+
+          Unix timestamp of the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+        - `lfs` (Bool, optional)
+
+          Fetch any [Git LFS](https://git-lfs.com/) files.
+
+          Default: `false`
+
+        - `ref` (String, optional)
+
+          By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
+
+          A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
+
+          Default: `"HEAD"`
+
+        - `rev` (String, optional)
+
+          A Git revision; a commit hash.
+
+          Default: the tip of `ref`
+
+        - `revCount` (Integer, optional)
+
+          Number of revisions in the history of the Git repository before the fetched commit.
+
+          If set, pass through the value to the output attribute set.
+          Otherwise, generated from the fetched Git tree.
+
+        - `shallow` (Bool, optional)
+
+          Make a shallow clone when fetching the Git tree.
+          When this is enabled, the options `ref` and `allRefs` have no effect anymore.
+
+          Default: `true`
+
+        - `submodules` (Bool, optional)
+
+          Also fetch submodules if available.
+
+          Default: `false`
+
+        - `url` (String, required)
+
+          The URL formats supported are the same as for Git itself.
 
           > **Example**
           >
-          > Fetch a GitHub repository using the attribute set representation:
-          >
           > ```nix
-          > builtins.fetchTree {
-          >   type = "github";
-          >   owner = "NixOS";
-          >   repo = "nixpkgs";
-          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-          > }
-          > ```
-          >
-          > This evaluates to the following attribute set:
-          >
-          > ```nix
-          > {
-          >   lastModified = 1686503798;
-          >   lastModifiedDate = "20230611171638";
-          >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
-          >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
-          >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
-          >   shortRev = "ae2e6b3";
+          > fetchTree {
+          >   type = "git";
+          >   url = "git@github.com:NixOS/nixpkgs.git";
           > }
           > ```
 
-          > **Example**
+          > **Note**
           >
-          > Fetch the same GitHub repository using the URL-like syntax:
-          >
-          >   ```nix
-          >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
-          >   ```
-        )"));
-        parse_document(*root, after, CMARK_OPT_DEFAULT);
+          > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
 
-        auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
-        assert(p);
-        return {&*p};
-    }(),
+      - `"tarball"`
+
+        Download a tar archive and extract it into the Nix store.
+        This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
+
+        - `url` (String, required)
+
+           > **Example**
+           >
+           > ```nix
+           > fetchTree {
+           >   type = "tarball";
+           >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
+           > }
+           > ```
+
+      The following input types are still subject to change:
+
+      - `"path"`
+      - `"github"`
+      - `"gitlab"`
+      - `"sourcehut"`
+      - `"mercurial"`
+
+     *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
+     The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
+
+      > **Example**
+      >
+      > Fetch a GitHub repository using the attribute set representation:
+      >
+      > ```nix
+      > builtins.fetchTree {
+      >   type = "github";
+      >   owner = "NixOS";
+      >   repo = "nixpkgs";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      > }
+      > ```
+      >
+      > This evaluates to the following attribute set:
+      >
+      > ```nix
+      > {
+      >   lastModified = 1686503798;
+      >   lastModifiedDate = "20230611171638";
+      >   narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
+      >   outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
+      >   rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
+      >   shortRev = "ae2e6b3";
+      > }
+      > ```
+
+      > **Example**
+      >
+      > Fetch the same GitHub repository using the URL-like syntax:
+      >
+      >   ```nix
+      >   builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
+      >   ```
+    )"_sds,
     .fun = prim_fetchTree,
     .experimentalFeature = Xp::FetchTree,
 });
@@ -498,10 +583,10 @@ static void fetch(
     auto storePath = unpack ? fetchToStore(
                                   state.fetchSettings,
                                   *state.store,
-                                  fetchers::downloadTarball(*state.store, state.fetchSettings, *url),
+                                  fetchers::downloadTarball(state.store, state.fetchSettings, *url),
                                   FetchMode::Copy,
                                   name)
-                            : fetchers::downloadFile(*state.store, state.fetchSettings, *url, name).storePath;
+                            : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash
@@ -533,7 +618,7 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value ** args, Va
 static RegisterPrimOp primop_fetchurl({
     .name = "__fetchurl",
     .args = {"arg"},
-    .doc = R"(
+    .doc = &R"(
       Download the specified URL and return the path of the downloaded file.
       `arg` can be either a string denoting the URL, or an attribute set with the following attributes:
 
@@ -547,7 +632,7 @@ static RegisterPrimOp primop_fetchurl({
         characters that are invalid for the store.
 
       Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
-    )",
+    )"_sds,
     .fun = prim_fetchurl,
 });
 
@@ -559,7 +644,7 @@ static void prim_fetchTarball(EvalState & state, const PosIdx pos, Value ** args
 static RegisterPrimOp primop_fetchTarball({
     .name = "fetchTarball",
     .args = {"args"},
-    .doc = R"(
+    .doc = &R"(
       Download the specified URL, unpack it and return the path of the
       unpacked tree. The file must be a tape archive (`.tar`) compressed
       with `gzip`, `bzip2` or `xz`. If the tarball consists of a
@@ -597,7 +682,7 @@ static RegisterPrimOp primop_fetchTarball({
       ```
 
       Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval).
-    )",
+    )"_sds,
     .fun = prim_fetchTarball,
 });
 
@@ -610,7 +695,7 @@ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value ** args, Va
 static RegisterPrimOp primop_fetchGit({
     .name = "fetchGit",
     .args = {"args"},
-    .doc = R"(
+    .doc = &R"(
       Fetch a path from git. *args* can be a URL, in which case the HEAD
       of the repo at that URL is fetched. Otherwise, it can be an
       attribute with the following attributes (all except `url` optional):
@@ -813,7 +898,7 @@ static RegisterPrimOp primop_fetchGit({
       given, `fetchGit` uses the current content of the checked-out
       files, even if they are not committed or added to Git's index. It
       only considers files added to the Git repository, as listed by `git ls-files`.
-    )",
+    )"_sds,
     .fun = prim_fetchGit,
 });
 

src/libexpr/primops/fromTOML.cc

@@ -1,6 +1,6 @@
 #include "nix/expr/primops.hh"
 #include "nix/expr/eval-inline.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include "expr-config-private.hh"
 
@@ -170,10 +170,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va
     }
 }
 
-static RegisterPrimOp primop_fromTOML(
-    {.name = "fromTOML",
-     .args = {"e"},
-     .doc = R"(
+static RegisterPrimOp primop_fromTOML({
+    .name = "fromTOML",
+    .args = {"e"},
+    .doc = &R"(
       Convert a TOML string to a Nix value. For example,
 
       ```nix
@@ -186,7 +186,8 @@ static RegisterPrimOp primop_fromTOML(
       ```
 
       returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`.
-    )",
-     .fun = prim_fromTOML});
+    )"_sds,
+    .fun = prim_fromTOML,
+});
 
 } // namespace nix

src/libfetchers-tests/git.cc

@@ -196,7 +196,7 @@ TEST_F(GitTest, submodulePeriodSupport)
             {"ref", "main"},
         });
 
-    auto [accessor, i] = input.getAccessor(settings, *store);
+    auto [accessor, i] = input.getAccessor(settings, store);
 
     ASSERT_EQ(accessor->readFile(CanonPath("deps/sub/lib.txt")), "hello from submodule\n");
 }

src/libfetchers/fetchers.cc

@@ -6,7 +6,6 @@
 #include "nix/fetchers/fetch-settings.hh"
 #include "nix/fetchers/fetch-to-store.hh"
 #include "nix/util/url.hh"
-#include "nix/util/archive.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -27,9 +26,18 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
         throw Error("Input scheme with name %s already registered", schemeName);
 }
 
-const InputSchemeMap & getAllInputSchemes()
+nlohmann::json dumpRegisterInputSchemeInfo()
 {
-    return inputSchemes();
+    using nlohmann::json;
+
+    auto res = json::object();
+
+    for (auto & [name, scheme] : inputSchemes()) {
+        auto & r = res[name] = json::object();
+        r["allowedAttrs"] = scheme->allowedAttrs();
+    }
+
+    return res;
 }
 
 Input Input::fromURL(const Settings & settings, const std::string & url, bool requireTree)
@@ -111,7 +119,7 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs)
     return std::move(*res);
 }
 
-std::optional<std::string> Input::getFingerprint(Store & store) const
+std::optional<std::string> Input::getFingerprint(ref<Store> store) const
 {
     if (!scheme)
         return std::nullopt;
@@ -190,7 +198,7 @@ bool Input::contains(const Input & other) const
 }
 
 // FIXME: remove
-std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store & store) const
+std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, ref<Store> store) const
 {
     if (!scheme)
         throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@@ -200,9 +208,9 @@ std::pair<StorePath, Input> Input::fetchToStore(const Settings & settings, Store
             auto [accessor, result] = getAccessorUnchecked(settings, store);
 
             auto storePath =
-                nix::fetchToStore(settings, store, SourcePath(accessor), FetchMode::Copy, result.getName());
+                nix::fetchToStore(settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName());
 
-            auto narHash = store.queryPathInfo(storePath)->narHash;
+            auto narHash = store->queryPathInfo(storePath)->narHash;
             result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
             result.attrs.insert_or_assign("__final", Explicit<bool>(true));
@@ -289,7 +297,7 @@ void Input::checkLocks(Input specified, Input & result)
     }
 }
 
-std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, Store & store) const
+std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settings, ref<Store> store) const
 {
     try {
         auto [accessor, result] = getAccessorUnchecked(settings, store);
@@ -305,7 +313,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessor(const Settings & settin
     }
 }
 
-std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, Store & store) const
+std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings & settings, ref<Store> store) const
 {
     // FIXME: cache the accessor
 
@@ -325,13 +333,13 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
     */
     if (isFinal() && getNarHash()) {
         try {
-            auto storePath = computeStorePath(store);
+            auto storePath = computeStorePath(*store);
 
-            store.ensurePath(storePath);
+            store->ensurePath(storePath);
 
-            debug("using substituted/cached input '%s' in '%s'", to_string(), store.printStorePath(storePath));
+            debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath));
 
-            auto accessor = store.requireStoreObjectAccessor(storePath);
+            auto accessor = store->requireStoreObjectAccessor(storePath);
 
             accessor->fingerprint = getFingerprint(store);
 
@@ -341,7 +349,7 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
             if (accessor->fingerprint) {
                 ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive;
                 auto cacheKey = makeFetchToStoreCacheKey(getName(), *accessor->fingerprint, method, "/");
-                settings.getCache()->upsert(cacheKey, store, {}, storePath);
+                settings.getCache()->upsert(cacheKey, *store, {}, storePath);
             }
 
             accessor->setPathDisplay("«" + to_string() + "»");
@@ -369,10 +377,10 @@ Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash>
     return scheme->applyOverrides(*this, ref, rev);
 }
 
-void Input::clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const
+void Input::clone(const Settings & settings, const Path & destDir) const
 {
     assert(scheme);
-    scheme->clone(settings, store, *this, destDir);
+    scheme->clone(settings, *this, destDir);
 }
 
 std::optional<std::filesystem::path> Input::getSourcePath() const
@@ -485,19 +493,9 @@ void InputScheme::putFile(
     throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path);
 }
 
-void InputScheme::clone(
-    const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const
+void InputScheme::clone(const Settings & settings, const Input & input, const Path & destDir) const
 {
-    if (std::filesystem::exists(destDir))
-        throw Error("cannot clone into existing path %s", destDir);
-
-    auto [accessor, input2] = getAccessor(settings, store, input);
-
-    Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir));
-
-    auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); });
-
-    restorePath(destDir, *source);
+    throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const

src/libfetchers/git-lfs-fetch.cc

@@ -209,7 +209,7 @@ std::vector<nlohmann::json> Fetch::fetchUrls(const std::vector<Pointer> & pointe
     auto url = api.endpoint + "/objects/batch";
     const auto & authHeader = api.authHeader;
     FileTransferRequest request(parseURL(url));
-    request.method = HttpMethod::Post;
+    request.method = HttpMethod::POST;
     Headers headers;
     if (authHeader.has_value())
         headers.push_back({"Authorization", *authHeader});

src/libfetchers/git.cc

@@ -194,183 +194,28 @@ struct GitInputScheme : InputScheme
         return "git";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        return stripIndentation(R"(
-          Fetch a Git tree and copy it to the Nix store.
-          This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
-        )");
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "url",
-                {
-                    .type = "String",
-                    .required = true,
-                    .doc = R"(
-                      The URL formats supported are the same as for Git itself.
-
-                      > **Example**
-                      >
-                      > ```nix
-                      > fetchTree {
-                      >   type = "git";
-                      >   url = "git@github.com:NixOS/nixpkgs.git";
-                      > }
-                      > ```
-
-                      > **Note**
-                      >
-                      > If the URL points to a local directory, and no `ref` or `rev` is given, Nix only considers files added to the Git index, as listed by `git ls-files` but uses the *current file contents* of the Git working directory.
-                    )",
-                },
-            },
-            {
-                "ref",
-                {
-                    .type = "String",
-                    .required = false,
-                    .doc = R"(
-                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-                      A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
-
-                      Default: `"HEAD"`
-                    )",
-                },
-            },
-            {
-                "rev",
-                {
-                    .type = "String",
-                    .required = false,
-                    .doc = R"(
-                      A Git revision; a commit hash.
-
-                      Default: the tip of `ref`
-                    )",
-                },
-            },
-            {
-                "shallow",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      Make a shallow clone when fetching the Git tree.
-                      When this is enabled, the options `ref` and `allRefs` have no effect anymore.
-
-                      Default: `true`
-                    )",
-                },
-            },
-            {
-                "submodules",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      Also fetch submodules if available.
-
-                      Default: `false`
-                    )",
-                },
-            },
-            {
-                "lfs",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      Fetch any [Git LFS](https://git-lfs.com/) files.
-
-                      Default: `false`
-                    )",
-                },
-            },
-            {
-                "exportIgnore",
-                {},
-            },
-            {
-                "lastModified",
-                {
-                    .type = "Integer",
-                    .required = false,
-                    .doc = R"(
-                      Unix timestamp of the fetched commit.
-
-                      If set, pass through the value to the output attribute set.
-                      Otherwise, generated from the fetched Git tree.
-                    )",
-                },
-            },
-            {
-                "revCount",
-                {
-                    .type = "Integer",
-                    .required = false,
-                    .doc = R"(
-                      Number of revisions in the history of the Git repository before the fetched commit.
-
-                      If set, pass through the value to the output attribute set.
-                      Otherwise, generated from the fetched Git tree.
-                    )",
-                },
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "allRefs",
-                {
-                    .type = "Bool",
-                    .required = false,
-                    .doc = R"(
-                      By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
-
-                      Whether to fetch all references (eg. branches and tags) of the repository.
-                      With this argument being true, it's possible to load a `rev` from *any* `ref`.
-                      (Without setting this option, only `rev`s from the specified `ref` are supported).
-
-                      Default: `false`
-                    )",
-                },
-            },
-            {
-                "name",
-                {},
-            },
-            {
-                "dirtyRev",
-                {},
-            },
-            {
-                "dirtyShortRev",
-                {},
-            },
-            {
-                "verifyCommit",
-                {},
-            },
-            {
-                "keytype",
-                {},
-            },
-            {
-                "publicKey",
-                {},
-            },
-            {
-                "publicKeys",
-                {},
-            },
+        return {
+            "url",
+            "ref",
+            "rev",
+            "shallow",
+            "submodules",
+            "lfs",
+            "exportIgnore",
+            "lastModified",
+            "revCount",
+            "narHash",
+            "allRefs",
+            "name",
+            "dirtyRev",
+            "dirtyShortRev",
+            "verifyCommit",
+            "keytype",
+            "publicKey",
+            "publicKeys",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -433,8 +278,7 @@ struct GitInputScheme : InputScheme
         return res;
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto repoInfo = getRepoInfo(input);
 
@@ -779,7 +623,7 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
+    getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
     {
         assert(!repoInfo.workdirInfo.isDirty);
 
@@ -953,7 +797,7 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessorFromWorkdir(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
+    getAccessorFromWorkdir(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
     {
         auto repoPath = repoInfo.getPath().value();
 
@@ -1037,7 +881,7 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
 
@@ -1059,7 +903,7 @@ struct GitInputScheme : InputScheme
         return {accessor, std::move(final)};
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         auto makeFingerprint = [&](const Hash & rev) {
             return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")

src/libfetchers/github.cc

@@ -110,43 +110,18 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
+    StringSet allowedAttrs() const override
     {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "owner",
-                {},
-            },
-            {
-                "repo",
-                {},
-            },
-            {
-                "ref",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "lastModified",
-                {},
-            },
-            {
-                "host",
-                {},
-            },
-            {
-                "treeHash",
-                {},
-            },
+        return {
+            "owner",
+            "repo",
+            "ref",
+            "rev",
+            "narHash",
+            "lastModified",
+            "host",
+            "treeHash",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const fetchers::Settings & settings, const Attrs & attrs) const override
@@ -258,7 +233,7 @@ struct GitArchiveInputScheme : InputScheme
         std::optional<Hash> treeHash;
     };
 
-    virtual RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const = 0;
+    virtual RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const = 0;
 
     virtual DownloadUrl getDownloadUrl(const Settings & settings, const Input & input) const = 0;
 
@@ -268,7 +243,7 @@ struct GitArchiveInputScheme : InputScheme
         time_t lastModified;
     };
 
-    std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, Store & store, Input input) const
+    std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, ref<Store> store, Input input) const
     {
         if (!maybeGetStrAttr(input.attrs, "ref"))
             input.attrs.insert_or_assign("ref", "HEAD");
@@ -341,7 +316,7 @@ struct GitArchiveInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         auto [input, tarballInfo] = downloadArchive(settings, store, _input);
 
@@ -370,7 +345,7 @@ struct GitArchiveInputScheme : InputScheme
         return Xp::Flakes;
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())
             return rev->gitRev();
@@ -386,12 +361,6 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return "github";
     }
 
-    std::string schemeDescription() const override
-    {
-        // TODO
-        return "";
-    }
-
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Github supports PAT/OAuth2 tokens and HTTP Basic
@@ -418,7 +387,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return getStrAttr(input.attrs, "repo");
     }
 
-    RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override
+    RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
     {
         auto host = getHost(input);
         auto url = fmt(
@@ -432,7 +401,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
         auto downloadResult = downloadFile(store, settings, url, "source", headers);
         auto json = nlohmann::json::parse(
-            store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
+            store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         return RefInfo{
             .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1),
@@ -457,13 +426,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
         return DownloadUrl{parseURL(url), headers};
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto host = getHost(input);
         Input::fromURL(settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input)))
             .applyOverrides(input.getRef(), input.getRev())
-            .clone(settings, store, destDir);
+            .clone(settings, destDir);
     }
 };
 
@@ -474,12 +442,6 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return "gitlab";
     }
 
-    std::string schemeDescription() const override
-    {
-        // TODO
-        return "";
-    }
-
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // Gitlab supports 4 kinds of authorization, two of which are
@@ -499,7 +461,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return std::make_pair(token.substr(0, fldsplit), token.substr(fldsplit + 1));
     }
 
-    RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override
+    RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // See rate limiting note below
@@ -514,7 +476,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
 
         auto downloadResult = downloadFile(store, settings, url, "source", headers);
         auto json = nlohmann::json::parse(
-            store.requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
+            store->requireStoreObjectAccessor(downloadResult.storePath)->readFile(CanonPath::root));
 
         if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) {
             return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)};
@@ -545,8 +507,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         return DownloadUrl{parseURL(url), headers};
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
         // FIXME: get username somewhere
@@ -554,7 +515,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             settings,
             fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef(), input.getRev())
-            .clone(settings, store, destDir);
+            .clone(settings, destDir);
     }
 };
 
@@ -565,12 +526,6 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         return "sourcehut";
     }
 
-    std::string schemeDescription() const override
-    {
-        // TODO
-        return "";
-    }
-
     std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
     {
         // SourceHut supports both PAT and OAuth2. See
@@ -581,7 +536,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         // Once it is implemented, however, should work as expected.
     }
 
-    RefInfo getRevFromRef(const Settings & settings, nix::Store & store, const Input & input) const override
+    RefInfo getRevFromRef(const Settings & settings, nix::ref<Store> store, const Input & input) const override
     {
         // TODO: In the future, when the sourcehut graphql API is implemented for mercurial
         // and with anonymous access, this method should use it instead.
@@ -597,7 +552,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::string refUri;
         if (ref == "HEAD") {
             auto downloadFileResult = downloadFile(store, settings, fmt("%s/HEAD", base_url), "source", headers);
-            auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
+            auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
 
             auto remoteLine = git::parseLsRemoteLine(getLine(contents).first);
             if (!remoteLine) {
@@ -610,7 +565,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::regex refRegex(refUri);
 
         auto downloadFileResult = downloadFile(store, settings, fmt("%s/info/refs", base_url), "source", headers);
-        auto contents = store.requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
+        auto contents = store->requireStoreObjectAccessor(downloadFileResult.storePath)->readFile(CanonPath::root);
         std::istringstream is(contents);
 
         std::string line;
@@ -641,15 +596,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         return DownloadUrl{parseURL(url), headers};
     }
 
-    void clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir)
-        const override
+    void clone(const Settings & settings, const Input & input, const Path & destDir) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
         Input::fromURL(
             settings,
             fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
             .applyOverrides(input.getRef(), input.getRev())
-            .clone(settings, store, destDir);
+            .clone(settings, destDir);
     }
 };
 

src/libfetchers/include/nix/fetchers/fetchers.hh

@@ -113,7 +113,7 @@ public:
      * Fetch the entire input into the Nix store, returning the
      * location in the Nix store and the locked input.
      */
-    std::pair<StorePath, Input> fetchToStore(const Settings & settings, Store & store) const;
+    std::pair<StorePath, Input> fetchToStore(const Settings & settings, ref<Store> store) const;
 
     /**
      * Check the locking attributes in `result` against
@@ -133,17 +133,17 @@ public:
      * input without copying it to the store. Also return a possibly
      * unlocked input.
      */
-    std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, Store & store) const;
+    std::pair<ref<SourceAccessor>, Input> getAccessor(const Settings & settings, ref<Store> store) const;
 
 private:
 
-    std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, Store & store) const;
+    std::pair<ref<SourceAccessor>, Input> getAccessorUnchecked(const Settings & settings, ref<Store> store) const;
 
 public:
 
     Input applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const;
 
-    void clone(const Settings & settings, Store & store, const std::filesystem::path & destDir) const;
+    void clone(const Settings & settings, const Path & destDir) const;
 
     std::optional<std::filesystem::path> getSourcePath() const;
 
@@ -173,7 +173,7 @@ public:
      *
      * This is not a stable identifier between Nix versions, but not guaranteed to change either.
      */
-    std::optional<std::string> getFingerprint(Store & store) const;
+    std::optional<std::string> getFingerprint(ref<Store> store) const;
 };
 
 /**
@@ -203,34 +203,20 @@ struct InputScheme
      */
     virtual std::string_view schemeName() const = 0;
 
-    /**
-     * Longform description of this scheme, for documentation purposes.
-     */
-    virtual std::string schemeDescription() const = 0;
-
-    // TODO remove these defaults
-    struct AttributeInfo
-    {
-        const char * type = "String";
-        bool required = true;
-        const char * doc = "";
-    };
-
     /**
      * Allowed attributes in an attribute set that is converted to an
-     * input, and documentation for each attribute.
+     * input.
      *
-     * `type` is not included from this map, because the `type` field is
+     * `type` is not included from this set, because the `type` field is
       parsed first to choose which scheme; `type` is always required.
      */
-    virtual const std::map<std::string, AttributeInfo> & allowedAttrs() const = 0;
+    virtual StringSet allowedAttrs() const = 0;
 
     virtual ParsedURL toURL(const Input & input) const;
 
     virtual Input applyOverrides(const Input & input, std::optional<std::string> ref, std::optional<Hash> rev) const;
 
-    virtual void
-    clone(const Settings & settings, Store & store, const Input & input, const std::filesystem::path & destDir) const;
+    virtual void clone(const Settings & settings, const Input & input, const Path & destDir) const;
 
     virtual std::optional<std::filesystem::path> getSourcePath(const Input & input) const;
 
@@ -241,7 +227,7 @@ struct InputScheme
         std::optional<std::string> commitMsg) const;
 
     virtual std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & input) const = 0;
+    getAccessor(const Settings & settings, ref<Store> store, const Input & input) const = 0;
 
     /**
      * Is this `InputScheme` part of an experimental feature?
@@ -253,7 +239,7 @@ struct InputScheme
         return true;
     }
 
-    virtual std::optional<std::string> getFingerprint(Store & store, const Input & input) const
+    virtual std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const
     {
         return std::nullopt;
     }
@@ -277,12 +263,7 @@ struct InputScheme
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
 
-using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
-
-/**
- * Use this for docs, not for finding a specific scheme
- */
-const InputSchemeMap & getAllInputSchemes();
+nlohmann::json dumpRegisterInputSchemeInfo();
 
 struct PublicKey
 {

src/libfetchers/include/nix/fetchers/input-cache.hh

@@ -16,7 +16,7 @@ struct InputCache
     };
 
     CachedResult
-    getAccessor(const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries);
+    getAccessor(const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries);
 
     struct CachedInput
     {

src/libfetchers/include/nix/fetchers/registry.hh

@@ -57,7 +57,7 @@ std::shared_ptr<Registry> getCustomRegistry(const Settings & settings, const Pat
 
 Path getUserRegistryPath();
 
-Registries getRegistries(const Settings & settings, Store & store);
+Registries getRegistries(const Settings & settings, ref<Store> store);
 
 void overrideRegistry(const Settings & settings, const Input & from, const Input & to, const Attrs & extraAttrs);
 
@@ -72,6 +72,6 @@ enum class UseRegistries : int {
  * use the registries for which the filter function returns true.
  */
 std::pair<Input, Attrs>
-lookupInRegistries(const Settings & settings, Store & store, const Input & input, UseRegistries useRegistries);
+lookupInRegistries(const Settings & settings, ref<Store> store, const Input & input, UseRegistries useRegistries);
 
 } // namespace nix::fetchers

src/libfetchers/include/nix/fetchers/tarball.hh

@@ -25,7 +25,7 @@ struct DownloadFileResult
 };
 
 DownloadFileResult downloadFile(
-    Store & store,
+    ref<Store> store,
     const Settings & settings,
     const std::string & url,
     const std::string & name,
@@ -43,6 +43,6 @@ struct DownloadTarballResult
  * Download and import a tarball into the Git cache. The result is the
  * Git tree hash of the root directory.
  */
-ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url);
+ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url);
 
 } // namespace nix::fetchers

src/libfetchers/indirect.cc

@@ -60,33 +60,14 @@ struct IndirectInputScheme : InputScheme
         return "indirect";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        // TODO
-        return "";
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "id",
-                {},
-            },
-            {
-                "ref",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
+        return {
+            "id",
+            "ref",
+            "rev",
+            "narHash",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -126,7 +107,7 @@ struct IndirectInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & input) const override
     {
         throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
     }

src/libfetchers/input-cache.cc

@@ -6,7 +6,7 @@
 namespace nix::fetchers {
 
 InputCache::CachedResult InputCache::getAccessor(
-    const Settings & settings, Store & store, const Input & originalInput, UseRegistries useRegistries)
+    const Settings & settings, ref<Store> store, const Input & originalInput, UseRegistries useRegistries)
 {
     auto fetched = lookup(originalInput);
     Input resolvedInput = originalInput;

src/libfetchers/mercurial.cc

@@ -68,41 +68,16 @@ struct MercurialInputScheme : InputScheme
         return "hg";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        // TODO
-        return "";
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "url",
-                {},
-            },
-            {
-                "ref",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "revCount",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "name",
-                {},
-            },
+        return {
+            "url",
+            "ref",
+            "rev",
+            "revCount",
+            "narHash",
+            "name",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -179,7 +154,7 @@ struct MercurialInputScheme : InputScheme
         return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()};
     }
 
-    StorePath fetchToStore(const Settings & settings, Store & store, Input & input) const
+    StorePath fetchToStore(const Settings & settings, ref<Store> store, Input & input) const
     {
         auto origRev = input.getRev();
 
@@ -230,7 +205,7 @@ struct MercurialInputScheme : InputScheme
                     return files.count(file);
                 };
 
-                auto storePath = store.addToStore(
+                auto storePath = store->addToStore(
                     input.getName(),
                     {getFSSourceAccessor(), CanonPath(actualPath)},
                     ContentAddressMethod::Raw::NixArchive,
@@ -251,7 +226,7 @@ struct MercurialInputScheme : InputScheme
                     "Hash '%s' is not supported by Mercurial. Only sha1 is supported.",
                     rev.to_string(HashFormat::Base16, true));
 
-            return Cache::Key{"hgRev", {{"store", store.storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
+            return Cache::Key{"hgRev", {{"store", store->storeDir}, {"name", name}, {"rev", input.getRev()->gitRev()}}};
         };
 
         auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath {
@@ -271,7 +246,7 @@ struct MercurialInputScheme : InputScheme
 
         /* If we have a rev, check if we have a cached store path. */
         if (auto rev = input.getRev()) {
-            if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), store))
+            if (auto res = settings.getCache()->lookupStorePath(revInfoKey(*rev), *store))
                 return makeResult(res->value, res->storePath);
         }
 
@@ -325,7 +300,7 @@ struct MercurialInputScheme : InputScheme
 
         /* Now that we have the rev, check the cache again for a
            cached store path. */
-        if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), store))
+        if (auto res = settings.getCache()->lookupStorePath(revInfoKey(rev), *store))
             return makeResult(res->value, res->storePath);
 
         Path tmpDir = createTempDir();
@@ -335,7 +310,7 @@ struct MercurialInputScheme : InputScheme
 
         deletePath(tmpDir + "/.hg_archival.txt");
 
-        auto storePath = store.addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
+        auto storePath = store->addToStore(name, {getFSSourceAccessor(), CanonPath(tmpDir)});
 
         Attrs infoAttrs({
             {"revCount", (uint64_t) revCount},
@@ -344,18 +319,18 @@ struct MercurialInputScheme : InputScheme
         if (!origRev)
             settings.getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}});
 
-        settings.getCache()->upsert(revInfoKey(rev), store, infoAttrs, storePath);
+        settings.getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath);
 
         return makeResult(infoAttrs, std::move(storePath));
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
 
         auto storePath = fetchToStore(settings, store, input);
-        auto accessor = store.requireStoreObjectAccessor(storePath);
+        auto accessor = store->requireStoreObjectAccessor(storePath);
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 
@@ -367,7 +342,7 @@ struct MercurialInputScheme : InputScheme
         return (bool) input.getRev();
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto rev = input.getRev())
             return rev->gitRev();

src/libfetchers/path.cc

@@ -40,42 +40,20 @@ struct PathInputScheme : InputScheme
         return "path";
     }
 
-    std::string schemeDescription() const override
+    StringSet allowedAttrs() const override
     {
-        // TODO
-        return "";
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "path",
-                {},
-            },
+        return {
+            "path",
             /* Allow the user to pass in "fake" tree info
                attributes. This is useful for making a pinned tree work
                the same as the repository from which is exported (e.g.
                path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
              */
-            {
-                "rev",
-                {},
-            },
-            {
-                "revCount",
-                {},
-            },
-            {
-                "lastModified",
-                {},
-            },
-            {
-                "narHash",
-                {},
-            },
+            "rev",
+            "revCount",
+            "lastModified",
+            "narHash",
         };
-        return attrs;
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -139,7 +117,7 @@ struct PathInputScheme : InputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         Input input(_input);
         auto path = getStrAttr(input.attrs, "path");
@@ -147,31 +125,31 @@ struct PathInputScheme : InputScheme
         auto absPath = getAbsPath(input);
 
         // FIXME: check whether access to 'path' is allowed.
-        auto storePath = store.maybeParseStorePath(absPath.string());
+        auto storePath = store->maybeParseStorePath(absPath.string());
 
         if (storePath)
-            store.addTempRoot(*storePath);
+            store->addTempRoot(*storePath);
 
         time_t mtime = 0;
-        if (!storePath || storePath->name() != "source" || !store.isValidPath(*storePath)) {
+        if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath));
             // FIXME: try to substitute storePath.
             auto src = sinkToSource(
                 [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); });
-            storePath = store.addToStoreFromDump(*src, "source");
+            storePath = store->addToStoreFromDump(*src, "source");
         }
 
-        auto accessor = store.requireStoreObjectAccessor(*storePath);
+        auto accessor = store->requireStoreObjectAccessor(*storePath);
 
         // To prevent `fetchToStore()` copying the path again to Nix
         // store, pre-create an entry in the fetcher cache.
-        auto info = store.queryPathInfo(*storePath);
+        auto info = store->queryPathInfo(*storePath);
         accessor->fingerprint =
-            fmt("path:%s", store.queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
+            fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true));
         settings.getCache()->upsert(
             makeFetchToStoreCacheKey(
                 input.getName(), *accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"),
-            store,
+            *store,
             {},
             *storePath);
 

src/libfetchers/registry.cc

@@ -136,7 +136,7 @@ void overrideRegistry(const Settings & settings, const Input & from, const Input
     getFlagRegistry(settings)->add(from, to, extraAttrs);
 }
 
-static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, Store & store)
+static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, ref<Store> store)
 {
     static auto reg = [&]() {
         auto path = settings.flakeRegistry.get();
@@ -149,9 +149,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
             [&] -> SourcePath {
                 if (!isAbsolute(path)) {
                     auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath;
-                    if (auto store2 = dynamic_cast<LocalFSStore *>(&store))
+                    if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                         store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json");
-                    return {store.requireStoreObjectAccessor(storePath)};
+                    return {store->requireStoreObjectAccessor(storePath)};
                 } else {
                     return SourcePath{getFSSourceAccessor(), CanonPath{path}}.resolveSymlinks();
                 }
@@ -162,7 +162,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(const Settings & settings, St
     return reg;
 }
 
-Registries getRegistries(const Settings & settings, Store & store)
+Registries getRegistries(const Settings & settings, ref<Store> store)
 {
     Registries registries;
     registries.push_back(getFlagRegistry(settings));
@@ -173,7 +173,7 @@ Registries getRegistries(const Settings & settings, Store & store)
 }
 
 std::pair<Input, Attrs>
-lookupInRegistries(const Settings & settings, Store & store, const Input & _input, UseRegistries useRegistries)
+lookupInRegistries(const Settings & settings, ref<Store> store, const Input & _input, UseRegistries useRegistries)
 {
     Attrs extraAttrs;
     int n = 0;

src/libfetchers/tarball.cc

@@ -13,7 +13,7 @@
 namespace nix::fetchers {
 
 DownloadFileResult downloadFile(
-    Store & store,
+    ref<Store> store,
     const Settings & settings,
     const std::string & url,
     const std::string & name,
@@ -28,7 +28,7 @@ DownloadFileResult downloadFile(
             {"name", name},
         }}};
 
-    auto cached = settings.getCache()->lookupStorePath(key, store);
+    auto cached = settings.getCache()->lookupStorePath(key, *store);
 
     auto useCached = [&]() -> DownloadFileResult {
         return {
@@ -74,7 +74,7 @@ DownloadFileResult downloadFile(
         dumpString(res.data, sink);
         auto hash = hashString(HashAlgorithm::SHA256, res.data);
         auto info = ValidPathInfo::makeFromCA(
-            store,
+            *store,
             name,
             FixedOutputInfo{
                 .method = FileIngestionMethod::Flat,
@@ -84,7 +84,7 @@ DownloadFileResult downloadFile(
             hashString(HashAlgorithm::SHA256, sink.s));
         info.narSize = sink.s.size();
         auto source = StringSource{sink.s};
-        store.addToStore(info, source, NoRepair, NoCheckSigs);
+        store->addToStore(info, source, NoRepair, NoCheckSigs);
         storePath = std::move(info.path);
     }
 
@@ -93,7 +93,7 @@ DownloadFileResult downloadFile(
         key.second.insert_or_assign("url", url);
         assert(!res.urls.empty());
         infoAttrs.insert_or_assign("url", *res.urls.rbegin());
-        settings.getCache()->upsert(key, store, infoAttrs, *storePath);
+        settings.getCache()->upsert(key, *store, infoAttrs, *storePath);
     }
 
     return {
@@ -214,7 +214,7 @@ static DownloadTarballResult downloadTarball_(
     return attrsToResult(infoAttrs);
 }
 
-ref<SourceAccessor> downloadTarball(Store & store, const Settings & settings, const std::string & url)
+ref<SourceAccessor> downloadTarball(ref<Store> store, const Settings & settings, const std::string & url)
 {
     /* Go through Input::getAccessor() to ensure that the resulting
        accessor has a fingerprint. */
@@ -278,7 +278,7 @@ struct CurlInputScheme : InputScheme
            HTTP request. Now that we've processed the Nix-specific
            attributes above, remove them so we don't also send them as
            part of the HTTP request. */
-        for (auto & [param, _] : allowedAttrs())
+        for (auto & param : allowedAttrs())
             url.query.erase(param);
 
         input.attrs.insert_or_assign("type", std::string{schemeName()});
@@ -286,83 +286,18 @@ struct CurlInputScheme : InputScheme
         return input;
     }
 
-    static const std::map<std::string, AttributeInfo> & allowedAttrsImpl()
+    StringSet allowedAttrs() const override
     {
-        static const std::map<std::string, AttributeInfo> attrs = {
-            {
-                "url",
-                {
-                    .type = "String",
-                    .required = true,
-                    .doc = R"(
-                      Supported protocols:
-
-                      - `https`
-
-                        > **Example**
-                        >
-                        > ```nix
-                        > fetchTree {
-                        >   type = "file";
-                        >   url = "https://example.com/index.html";
-                        > }
-                        > ```
-
-                      - `http`
-
-                        Insecure HTTP transfer for legacy sources.
-
-                        > **Warning**
-                        >
-                        > HTTP performs no encryption or authentication.
-                        > Use a `narHash` known in advance to ensure the output has expected contents.
-
-                      - `file`
-
-                        A file on the local file system.
-
-                        > **Example**
-                        >
-                        > ```nix
-                        > fetchTree {
-                        >   type = "file";
-                        >   url = "file:///home/eelco/nix/README.md";
-                        > }
-                        > ```
-                    )",
-                },
-            },
-            {
-                "narHash",
-                {},
-            },
-            {
-                "name",
-                {},
-            },
-            {
-                "unpack",
-                {},
-            },
-            {
-                "rev",
-                {},
-            },
-            {
-                "revCount",
-                {},
-            },
-            {
-                "lastModified",
-                {},
-            },
+        return {
+            "type",
+            "url",
+            "narHash",
+            "name",
+            "unpack",
+            "rev",
+            "revCount",
+            "lastModified",
         };
-        return attrs;
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        return allowedAttrsImpl();
     }
 
     std::optional<Input> inputFromAttrs(const Settings & settings, const Attrs & attrs) const override
@@ -397,14 +332,6 @@ struct FileInputScheme : CurlInputScheme
         return "file";
     }
 
-    std::string schemeDescription() const override
-    {
-        return stripIndentation(R"(
-          Place a plain file into the Nix store.
-          This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
-        )");
-    }
-
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -414,7 +341,7 @@ struct FileInputScheme : CurlInputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         auto input(_input);
 
@@ -424,10 +351,10 @@ struct FileInputScheme : CurlInputScheme
            tarballs. */
         auto file = downloadFile(store, settings, getStrAttr(input.attrs, "url"), input.getName());
 
-        auto narHash = store.queryPathInfo(file.storePath)->narHash;
+        auto narHash = store->queryPathInfo(file.storePath)->narHash;
         input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-        auto accessor = ref{store.getFSAccessor(file.storePath)};
+        auto accessor = ref{store->getFSAccessor(file.storePath)};
 
         accessor->setPathDisplay("«" + input.to_string() + "»");
 
@@ -442,34 +369,6 @@ struct TarballInputScheme : CurlInputScheme
         return "tarball";
     }
 
-    std::string schemeDescription() const override
-    {
-        return stripIndentation(R"(
-          Download a tar archive and extract it into the Nix store.
-          This has the same underlying implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
-        )");
-    }
-
-    const std::map<std::string, AttributeInfo> & allowedAttrs() const override
-    {
-        static const std::map<std::string, AttributeInfo> attrs = [] {
-            auto attrs = CurlInputScheme::allowedAttrsImpl();
-            // Override the "url" attribute to add tarball-specific example
-            attrs["url"].doc = R"(
-              > **Example**
-              >
-              > ```nix
-              > fetchTree {
-              >   type = "tarball";
-              >   url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
-              > }
-              > ```
-            )";
-            return attrs;
-        }();
-        return attrs;
-    }
-
     bool isValidURL(const ParsedURL & url, bool requireTree) const override
     {
         auto parsedUrlScheme = parseUrlScheme(url.scheme);
@@ -480,7 +379,7 @@ struct TarballInputScheme : CurlInputScheme
     }
 
     std::pair<ref<SourceAccessor>, Input>
-    getAccessor(const Settings & settings, Store & store, const Input & _input) const override
+    getAccessor(const Settings & settings, ref<Store> store, const Input & _input) const override
     {
         auto input(_input);
 
@@ -505,7 +404,7 @@ struct TarballInputScheme : CurlInputScheme
         return {result.accessor, input};
     }
 
-    std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
+    std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
         if (auto narHash = input.getNarHash())
             return narHash->to_string(HashFormat::SRI, true);

src/libflake/flake-primops.cc

@@ -19,6 +19,7 @@
 #include "nix/expr/eval-settings.hh"
 #include "nix/expr/symbol-table.hh"
 #include "nix/expr/value.hh"
+#include "nix/expr/string-data-static.hh"
 #include "nix/fetchers/attrs.hh"
 #include "nix/fetchers/fetchers.hh"
 #include "nix/util/configuration.hh"
@@ -62,7 +63,7 @@ PrimOp getFlake(const Settings & settings)
     return PrimOp{
         .name = "__getFlake",
         .args = {"args"},
-        .doc = R"(
+        .doc = &R"(
           Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
 
           ```nix
@@ -76,7 +77,7 @@ PrimOp getFlake(const Settings & settings)
           ```nix
           (builtins.getFlake "github:edolstra/dwarffs").rev
           ```
-        )",
+        )"_sds,
         .fun = prim_getFlake,
         .experimentalFeature = Xp::Flakes,
     };
@@ -104,7 +105,7 @@ static void prim_parseFlakeRef(EvalState & state, const PosIdx pos, Value ** arg
 nix::PrimOp parseFlakeRef({
     .name = "__parseFlakeRef",
     .args = {"flake-ref"},
-    .doc = R"(
+    .doc = &R"(
       Parse a flake reference, and return its exploded form.
 
       For example:
@@ -118,7 +119,7 @@ nix::PrimOp parseFlakeRef({
       ```nix
       { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; }
       ```
-    )",
+    )"_sds,
     .fun = prim_parseFlakeRef,
     .experimentalFeature = Xp::Flakes,
 });
@@ -162,7 +163,7 @@ static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value **
 nix::PrimOp flakeRefToString({
     .name = "__flakeRefToString",
     .args = {"attrs"},
-    .doc = R"(
+    .doc = &R"(
       Convert a flake reference from attribute set format to URL format.
 
       For example:
@@ -178,7 +179,7 @@ nix::PrimOp flakeRefToString({
       ```nix
       "github:NixOS/nixpkgs/23.05?dir=lib"
       ```
-    )",
+    )"_sds,
     .fun = prim_flakeRefToString,
     .experimentalFeature = Xp::Flakes,
 });

src/libflake/flake.cc

@@ -373,7 +373,7 @@ static Flake getFlake(
 {
     // Fetch a lazy tree first.
     auto cachedInput =
-        state.inputCache->getAccessor(state.fetchSettings, *state.store, originalRef.input, useRegistries);
+        state.inputCache->getAccessor(state.fetchSettings, state.store, originalRef.input, useRegistries);
 
     auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir);
     auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir);
@@ -390,7 +390,7 @@ static Flake getFlake(
         // FIXME: need to remove attrs that are invalidated by the changed input attrs, such as 'narHash'.
         newLockedRef.input.attrs.erase("narHash");
         auto cachedInput2 = state.inputCache->getAccessor(
-            state.fetchSettings, *state.store, newLockedRef.input, fetchers::UseRegistries::No);
+            state.fetchSettings, state.store, newLockedRef.input, fetchers::UseRegistries::No);
         cachedInput.accessor = cachedInput2.accessor;
         lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir);
     }
@@ -756,7 +756,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef,
                                     return {*resolvedPath, *input.ref};
                                 } else {
                                     auto cachedInput = state.inputCache->getAccessor(
-                                        state.fetchSettings, *state.store, input.ref->input, useRegistriesInputs);
+                                        state.fetchSettings, state.store, input.ref->input, useRegistriesInputs);
 
                                     auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir);
 
@@ -975,7 +975,7 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes)
     state.callFunction(*vCallFlake, args, vRes, noPos);
 }
 
-std::optional<Fingerprint> LockedFlake::getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const
+std::optional<Fingerprint> LockedFlake::getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const
 {
     if (lockFile.isUnlocked(fetchSettings))
         return std::nullopt;
@@ -1005,7 +1005,7 @@ Flake::~Flake() {}
 ref<eval_cache::EvalCache> openEvalCache(EvalState & state, ref<const LockedFlake> lockedFlake)
 {
     auto fingerprint = state.settings.useEvalCache && state.settings.pureEval
-                           ? lockedFlake->getFingerprint(*state.store, state.fetchSettings)
+                           ? lockedFlake->getFingerprint(state.store, state.fetchSettings)
                            : std::nullopt;
     auto rootLoader = [&state, lockedFlake]() {
         /* For testing whether the evaluation cache is

src/libflake/flakeref.cc

@@ -64,8 +64,8 @@ std::ostream & operator<<(std::ostream & str, const FlakeRef & flakeRef)
     return str;
 }
 
-FlakeRef
-FlakeRef::resolve(const fetchers::Settings & fetchSettings, Store & store, fetchers::UseRegistries useRegistries) const
+FlakeRef FlakeRef::resolve(
+    const fetchers::Settings & fetchSettings, ref<Store> store, fetchers::UseRegistries useRegistries) const
 {
     auto [input2, extraAttrs] = lookupInRegistries(fetchSettings, store, input, useRegistries);
     return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
@@ -289,7 +289,7 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Settings & fetchSettings, const fet
 }
 
 std::pair<ref<SourceAccessor>, FlakeRef>
-FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const
+FlakeRef::lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const
 {
     auto [accessor, lockedInput] = input.getAccessor(fetchSettings, store);
     return {accessor, FlakeRef(std::move(lockedInput), subdir)};

src/libflake/include/nix/flake/flake.hh

@@ -135,7 +135,7 @@ struct LockedFlake
      */
     std::map<ref<Node>, SourcePath> nodePaths;
 
-    std::optional<Fingerprint> getFingerprint(Store & store, const fetchers::Settings & fetchSettings) const;
+    std::optional<Fingerprint> getFingerprint(ref<Store> store, const fetchers::Settings & fetchSettings) const;
 };
 
 struct LockFlags

src/libflake/include/nix/flake/flakeref.hh

@@ -73,12 +73,13 @@ struct FlakeRef
 
     FlakeRef resolve(
         const fetchers::Settings & fetchSettings,
-        Store & store,
+        ref<Store> store,
         fetchers::UseRegistries useRegistries = fetchers::UseRegistries::All) const;
 
     static FlakeRef fromAttrs(const fetchers::Settings & fetchSettings, const fetchers::Attrs & attrs);
 
-    std::pair<ref<SourceAccessor>, FlakeRef> lazyFetch(const fetchers::Settings & fetchSettings, Store & store) const;
+    std::pair<ref<SourceAccessor>, FlakeRef>
+    lazyFetch(const fetchers::Settings & fetchSettings, ref<Store> store) const;
 
     /**
      * Canonicalize a flakeref for the purpose of comparing "old" and

src/libstore-c/meson.build

@@ -35,8 +35,6 @@ include_dirs = [ include_directories('.') ]
 
 headers = files(
   'nix_api_store.h',
-  'nix_api_store/derivation.h',
-  'nix_api_store/store_path.h',
 )
 
 # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c`

src/libstore-c/nix_api_store.h

@@ -12,8 +12,6 @@
  */
 
 #include "nix_api_util.h"
-#include "nix_api_store/store_path.h"
-#include "nix_api_store/derivation.h"
 #include <stdbool.h>
 
 #ifdef __cplusplus
@@ -23,6 +21,10 @@ extern "C" {
 
 /** @brief Reference to a Nix store */
 typedef struct Store Store;
+/** @brief Nix store path */
+typedef struct StorePath StorePath;
+/** @brief Nix Derivation */
+typedef struct nix_derivation nix_derivation;
 
 /**
  * @brief Initializes the Nix store library
@@ -116,6 +118,30 @@ nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_ca
  */
 StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
 
+/**
+ * @brief Get the path name (e.g. "name" in /nix/store/...-name)
+ *
+ * @param[in] store_path the path to get the name from
+ * @param[in] callback called with the name
+ * @param[in] user_data arbitrary data, passed to the callback when it's called.
+ */
+void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
+
+/**
+ * @brief Copy a StorePath
+ *
+ * @param[in] p the path to copy
+ * @return a new StorePath
+ */
+StorePath * nix_store_path_clone(const StorePath * p);
+
+/** @brief Deallocate a StorePath
+ *
+ * Does not fail.
+ * @param[in] p the path to free
+ */
+void nix_store_path_free(StorePath * p);
+
 /**
  * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in
  * the store)
@@ -203,6 +229,14 @@ nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store
  */
 StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation);
 
+/**
+ * @brief Deallocate a `nix_derivation`
+ *
+ * Does not fail.
+ * @param[in] drv the derivation to free
+ */
+void nix_derivation_free(nix_derivation * drv);
+
 /**
  * @brief Copy the closure of `path` from `srcStore` to `dstStore`.
  *

src/libstore-c/nix_api_store/derivation.h

@@ -1,38 +0,0 @@
-#ifndef NIX_API_STORE_DERIVATION_H
-#define NIX_API_STORE_DERIVATION_H
-/**
- * @defgroup libstore_derivation Derivation
- * @ingroup libstore
- * @brief Derivation operations that don't require a Store
- * @{
- */
-/** @file
- * @brief Derivation operations
- */
-
-#include "nix_api_util.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-// cffi start
-
-/** @brief Nix Derivation */
-typedef struct nix_derivation nix_derivation;
-
-/**
- * @brief Deallocate a `nix_derivation`
- *
- * Does not fail.
- * @param[in] drv the derivation to free
- */
-void nix_derivation_free(nix_derivation * drv);
-
-// cffi end
-#ifdef __cplusplus
-}
-#endif
-/**
- * @}
- */
-#endif // NIX_API_STORE_DERIVATION_H

src/libstore-c/nix_api_store/store_path.h

@@ -1,54 +0,0 @@
-#ifndef NIX_API_STORE_STORE_PATH_H
-#define NIX_API_STORE_STORE_PATH_H
-/**
- * @defgroup libstore_storepath StorePath
- * @ingroup libstore
- * @brief Store path operations that don't require a Store
- * @{
- */
-/** @file
- * @brief Store path operations
- */
-
-#include "nix_api_util.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-// cffi start
-
-/** @brief Nix store path */
-typedef struct StorePath StorePath;
-
-/**
- * @brief Copy a StorePath
- *
- * @param[in] p the path to copy
- * @return a new StorePath
- */
-StorePath * nix_store_path_clone(const StorePath * p);
-
-/** @brief Deallocate a StorePath
- *
- * Does not fail.
- * @param[in] p the path to free
- */
-void nix_store_path_free(StorePath * p);
-
-/**
- * @brief Get the path name (e.g. "<name>" in /nix/store/<hash>-<name>)
- *
- * @param[in] store_path the path to get the name from
- * @param[in] callback called with the name
- * @param[in] user_data arbitrary data, passed to the callback when it's called.
- */
-void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
-
-// cffi end
-#ifdef __cplusplus
-}
-#endif
-/**
- * @}
- */
-#endif // NIX_API_STORE_STORE_PATH_H

src/libstore/filetransfer.cc

@@ -386,17 +386,17 @@ struct curlFileTransfer : public FileTransfer
             if (settings.downloadSpeed.get() > 0)
                 curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
 
-            if (request.method == HttpMethod::Head)
+            if (request.method == HttpMethod::HEAD)
                 curl_easy_setopt(req, CURLOPT_NOBODY, 1);
 
-            if (request.method == HttpMethod::Delete)
+            if (request.method == HttpMethod::DELETE)
                 curl_easy_setopt(req, CURLOPT_CUSTOMREQUEST, "DELETE");
 
             if (request.data) {
-                if (request.method == HttpMethod::Post) {
+                if (request.method == HttpMethod::POST) {
                     curl_easy_setopt(req, CURLOPT_POST, 1L);
                     curl_easy_setopt(req, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) request.data->sizeHint);
-                } else if (request.method == HttpMethod::Put) {
+                } else if (request.method == HttpMethod::PUT) {
                     curl_easy_setopt(req, CURLOPT_UPLOAD, 1L);
                     curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->sizeHint);
                 } else {

src/libstore/http-binary-cache-store.cc

@@ -120,7 +120,7 @@ bool HttpBinaryCacheStore::fileExists(const std::string & path)
 
     try {
         FileTransferRequest request(makeRequest(path));
-        request.method = HttpMethod::Head;
+        request.method = HttpMethod::HEAD;
         getFileTransfer()->download(request);
         return true;
     } catch (FileTransferError & e) {
@@ -141,7 +141,7 @@ void HttpBinaryCacheStore::upload(
     std::optional<Headers> headers)
 {
     auto req = makeRequest(path);
-    req.method = HttpMethod::Put;
+    req.method = HttpMethod::PUT;
 
     if (headers) {
         req.headers.reserve(req.headers.size() + headers->size());

src/libstore/include/nix/store/filetransfer.hh

@@ -87,11 +87,11 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT;
  * HTTP methods supported by FileTransfer.
  */
 enum struct HttpMethod {
-    Get,
-    Put,
-    Head,
-    Post,
-    Delete,
+    GET,
+    PUT,
+    HEAD,
+    POST,
+    DELETE,
 };
 
 /**
@@ -110,7 +110,7 @@ struct FileTransferRequest
     VerbatimURL uri;
     Headers headers;
     std::string expectedETag;
-    HttpMethod method = HttpMethod::Get;
+    HttpMethod method = HttpMethod::GET;
     size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT;
     ActivityId parentAct;
@@ -164,14 +164,14 @@ struct FileTransferRequest
     std::string verb() const
     {
         switch (method) {
-        case HttpMethod::Head:
-        case HttpMethod::Get:
+        case HttpMethod::HEAD:
+        case HttpMethod::GET:
             return "download";
-        case HttpMethod::Put:
-        case HttpMethod::Post:
+        case HttpMethod::PUT:
+        case HttpMethod::POST:
             assert(data);
             return "upload";
-        case HttpMethod::Delete:
+        case HttpMethod::DELETE:
             return "delet";
         }
         unreachable();

src/libstore/local-store.cc

@@ -1246,8 +1246,10 @@ StorePath LocalStore::addToStoreFromDump(
 
     auto desc = ContentAddressWithReferences::fromParts(
         hashMethod,
-        methodsMatch ? dumpHash
-                     : hashPath(makeFSSourceAccessor(tempPath), hashMethod.getFileIngestionMethod(), hashAlgo).first,
+        methodsMatch
+            ? dumpHash
+            : hashPath(PosixSourceAccessor::createAtRoot(tempPath), hashMethod.getFileIngestionMethod(), hashAlgo)
+                  .first,
         {
             .others = references,
             // caller is not capable of creating a self-reference, because this is content-addressed without modulus
@@ -1383,9 +1385,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
             checkInterrupt();
             auto name = link.path().filename();
             printMsg(lvlTalkative, "checking contents of %s", name);
-            std::string hash =
-                hashPath(makeFSSourceAccessor(link.path()), FileIngestionMethod::NixArchive, HashAlgorithm::SHA256)
-                    .first.to_string(HashFormat::Nix32, false);
+            std::string hash = hashPath(
+                                   PosixSourceAccessor::createAtRoot(link.path()),
+                                   FileIngestionMethod::NixArchive,
+                                   HashAlgorithm::SHA256)
+                                   .first.to_string(HashFormat::Nix32, false);
             if (hash != name.string()) {
                 printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash);
                 if (repair) {

src/libstore/optimise-store.cc

@@ -172,7 +172,7 @@ void LocalStore::optimisePath_(
         auto stLink = lstat(linkPath.string());
         if (st.st_size != stLink.st_size || (repair && hash != ({
                                                            hashPath(
-                                                               makeFSSourceAccessor(linkPath),
+                                                               PosixSourceAccessor::createAtRoot(linkPath),
                                                                FileSerialisationMethod::NixArchive,
                                                                HashAlgorithm::SHA256)
                                                                .hash;

src/libstore/s3-binary-cache-store.cc

@@ -295,7 +295,7 @@ std::string S3BinaryCacheStore::createMultipartUpload(
     url.query["uploads"] = "";
     req.uri = VerbatimURL(url);
 
-    req.method = HttpMethod::Post;
+    req.method = HttpMethod::POST;
     StringSource payload{std::string_view("")};
     req.data = {payload};
     req.mimeType = mimeType;
@@ -325,7 +325,7 @@ S3BinaryCacheStore::uploadPart(std::string_view key, std::string_view uploadId,
     }
 
     auto req = makeRequest(key);
-    req.method = HttpMethod::Put;
+    req.method = HttpMethod::PUT;
     req.setupForS3();
 
     auto url = req.uri.parsed();
@@ -355,7 +355,7 @@ void S3BinaryCacheStore::abortMultipartUpload(std::string_view key, std::string_
         auto url = req.uri.parsed();
         url.query["uploadId"] = uploadId;
         req.uri = VerbatimURL(url);
-        req.method = HttpMethod::Delete;
+        req.method = HttpMethod::DELETE;
 
         getFileTransfer()->enqueueFileTransfer(req).get();
     } catch (...) {
@@ -372,7 +372,7 @@ void S3BinaryCacheStore::completeMultipartUpload(
     auto url = req.uri.parsed();
     url.query["uploadId"] = uploadId;
     req.uri = VerbatimURL(url);
-    req.method = HttpMethod::Post;
+    req.method = HttpMethod::POST;
 
     std::string xml = "<CompleteMultipartUpload>";
     for (const auto & [idx, etag] : enumerate(partEtags)) {

src/libutil/include/nix/util/cmark-cpp.hh

@@ -1,81 +0,0 @@
-#pragma once
-///@file
-
-#include "types.hh"
-#include "util.hh"
-
-#include <cmark.h>
-
-namespace nix::cmark {
-
-using Node = struct cmark_node;
-using NodeType = cmark_node_type;
-using ListType = cmark_list_type;
-
-using Iter = struct cmark_iter;
-
-struct Deleter
-{
-    void operator()(Node * ptr)
-    {
-        cmark_node_free(ptr);
-    }
-
-    void operator()(Iter * ptr)
-    {
-        cmark_iter_free(ptr);
-    }
-};
-
-template<typename T>
-using UniquePtr = std::unique_ptr<Node, Deleter>;
-
-static inline void parse_document(Node & root, std::string_view s, int options)
-{
-    cmark_parser * parser = cmark_parser_new_with_mem_into_root(options, cmark_get_default_mem_allocator(), &root);
-    cmark_parser_feed(parser, s.data(), s.size());
-    (void) cmark_parser_finish(parser);
-    cmark_parser_free(parser);
-}
-
-static inline UniquePtr<Node> parse_document(std::string_view s, int options)
-{
-    return UniquePtr<Node>{cmark_parse_document(s.data(), s.size(), options)};
-}
-
-static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
-{
-    return std::unique_ptr<char, FreeDeleter>{cmark_render_commonmark(&root, options, width)};
-}
-
-static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
-{
-    return std::unique_ptr<char, FreeDeleter>{cmark_render_xml(&root, options)};
-}
-
-static inline UniquePtr<Node> node_new(NodeType type)
-{
-    return UniquePtr<Node>{cmark_node_new(type)};
-}
-
-/**
- * The parent takes ownership
- */
-static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
-{
-    auto status = (bool) cmark_node_append_child(&node, &*child);
-    assert(status);
-    return *child.release();
-}
-
-static inline bool node_set_literal(Node & node, const char * content)
-{
-    return (bool) cmark_node_set_literal(&node, content);
-}
-
-static inline bool node_set_list_type(Node & node, ListType type)
-{
-    return (bool) cmark_node_set_list_type(&node, type);
-}
-
-} // namespace nix::cmark

src/libutil/include/nix/util/meson.build

@@ -17,7 +17,6 @@ headers = files(
   'checked-arithmetic.hh',
   'chunked-vector.hh',
   'closure.hh',
-  'cmark-cpp.hh',
   'comparator.hh',
   'compression.hh',
   'compute-levels.hh',

src/libutil/include/nix/util/terminal.hh

@@ -4,15 +4,7 @@
 #include <limits>
 #include <string>
 
-#include "nix/util/file-descriptor.hh"
-
 namespace nix {
-
-/**
- * Determine whether \param fd is a terminal.
- */
-bool isTTY(Descriptor fd);
-
 /**
  * Determine whether ANSI escape sequences are appropriate for the
  * present output.

src/libutil/meson.build

@@ -110,9 +110,6 @@ deps_private += cpuid
 nlohmann_json = dependency('nlohmann_json', version : '>= 3.9')
 deps_public += nlohmann_json
 
-cmark = dependency('libcmark', required : true)
-deps_public += cmark
-
 cxx = meson.get_compiler('cpp')
 
 config_priv_h = configure_file(

src/libutil/package.nix

@@ -5,7 +5,6 @@
 
   boost,
   brotli,
-  cmark,
   libarchive,
   libblake3,
   libcpuid,
@@ -58,7 +57,6 @@ mkMesonLibrary (finalAttrs: {
 
   propagatedBuildInputs = [
     boost
-    cmark
     libarchive
     nlohmann_json
   ];

src/libutil/terminal.cc

@@ -64,16 +64,6 @@ inline std::pair<int, size_t> charWidthUTF8Helper(std::string_view s)
 
 namespace nix {
 
-bool isTTY(Descriptor fd)
-{
-#ifndef _WIN32
-    return isatty(fd);
-#else
-    DWORD mode;
-    return GetConsoleMode(fd, &mode);
-#endif
-}
-
 bool isTTY()
 {
     static const bool tty = isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"

src/nix/cat.cc

@@ -75,7 +75,7 @@ struct CmdCatNar : StoreCommand, MixCat
 
     void run(ref<Store> store) override
     {
-        AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY));
+        AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
         if (!fd)
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};

src/nix/dump-path.cc

@@ -1,14 +1,13 @@
 #include "nix/cmd/command.hh"
 #include "nix/store/store-api.hh"
 #include "nix/util/archive.hh"
-#include "nix/util/terminal.hh"
 
 using namespace nix;
 
 static FdSink getNarSink()
 {
     auto fd = getStandardOutput();
-    if (isTTY(fd))
+    if (isatty(fd))
         throw UsageError("refusing to write NAR to a terminal");
     return FdSink(std::move(fd));
 }

src/nix/flake-prefetch-inputs.cc

@@ -45,7 +45,7 @@ struct CmdFlakePrefetchInputs : FlakeCommand
             if (auto lockedNode = dynamic_cast<const LockedNode *>(&node)) {
                 try {
                     Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef));
-                    auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, *store).first;
+                    auto accessor = lockedNode->lockedRef.input.getAccessor(fetchSettings, store).first;
                     fetchToStore(
                         fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName());
                 } catch (Error & e) {

src/nix/flake.cc

@@ -240,7 +240,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
                 j["lastModified"] = *lastModified;
             j["path"] = storePath;
             j["locks"] = lockedFlake.lockFile.toJSON().first;
-            if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings))
+            if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings))
                 j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false);
             printJSON(j);
         } else {
@@ -260,7 +260,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON
                 logger->cout(
                     ANSI_BOLD "Last modified:" ANSI_NORMAL " %s",
                     std::put_time(std::localtime(&*lastModified), "%F %T"));
-            if (auto fingerprint = lockedFlake.getFingerprint(*store, fetchSettings))
+            if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings))
                 logger->cout(
                     ANSI_BOLD "Fingerprint:" ANSI_NORMAL "   %s", fingerprint->to_string(HashFormat::Base16, false));
 
@@ -1019,7 +1019,7 @@ struct CmdFlakeNew : CmdFlakeInitCommon
 
 struct CmdFlakeClone : FlakeCommand
 {
-    std::filesystem::path destDir;
+    Path destDir;
 
     std::string description() override
     {
@@ -1049,7 +1049,7 @@ struct CmdFlakeClone : FlakeCommand
         if (destDir.empty())
             throw Error("missing flag '--dest'");
 
-        getFlakeRef().resolve(fetchSettings, *store).input.clone(fetchSettings, *store, destDir);
+        getFlakeRef().resolve(fetchSettings, store).input.clone(fetchSettings, destDir);
     }
 };
 
@@ -1100,7 +1100,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs
                     std::optional<StorePath> storePath;
                     if (!(*inputNode)->lockedRef.input.isRelative()) {
                         storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store)
-                                           : (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, *store).first;
+                                           : (*inputNode)->lockedRef.input.fetchToStore(fetchSettings, store).first;
                         sources.insert(*storePath);
                     }
                     if (json) {
@@ -1498,8 +1498,8 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON
     void run(ref<Store> store) override
     {
         auto originalRef = getFlakeRef();
-        auto resolvedRef = originalRef.resolve(fetchSettings, *store);
-        auto [accessor, lockedRef] = resolvedRef.lazyFetch(getEvalState()->fetchSettings, *store);
+        auto resolvedRef = originalRef.resolve(fetchSettings, store);
+        auto [accessor, lockedRef] = resolvedRef.lazyFetch(getEvalState()->fetchSettings, store);
         auto storePath =
             fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName());
         auto hash = store->queryPathInfo(storePath)->narHash;

src/nix/ls.cc

@@ -145,7 +145,7 @@ struct CmdLsNar : Command, MixLs
 
     void run() override
     {
-        AutoCloseFD fd = toDescriptor(open(narPath.c_str(), O_RDONLY));
+        AutoCloseFD fd = open(narPath.c_str(), O_RDONLY);
         if (!fd)
             throw SysError("opening NAR file '%s'", narPath);
         auto source = FdSource{fd.get()};

src/nix/main.cc

@@ -193,38 +193,20 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
 
     std::string dumpCli()
     {
-        using nlohmann::json;
-
-        auto res = json::object();
+        auto res = nlohmann::json::object();
 
         res["args"] = toJSON();
 
-        {
-            auto & stores = res["stores"] = json::object();
-            for (auto & [storeName, implem] : Implementations::registered()) {
-                auto & j = stores[storeName];
-                j["doc"] = implem.doc;
-                j["uri-schemes"] = implem.uriSchemes;
-                j["settings"] = implem.getConfig()->toJSON();
-                j["experimentalFeature"] = implem.experimentalFeature;
-            }
+        auto stores = nlohmann::json::object();
+        for (auto & [storeName, implem] : Implementations::registered()) {
+            auto & j = stores[storeName];
+            j["doc"] = implem.doc;
+            j["uri-schemes"] = implem.uriSchemes;
+            j["settings"] = implem.getConfig()->toJSON();
+            j["experimentalFeature"] = implem.experimentalFeature;
         }
-
-        {
-            auto & fetchers = res["fetchers"] = json::object();
-
-            for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
-                auto & s = fetchers[schemeName] = json::object();
-                s["description"] = scheme->schemeDescription();
-                auto & attrs = s["allowedAttrs"] = json::object();
-                for (auto & [fieldName, field] : scheme->allowedAttrs()) {
-                    auto & f = attrs[fieldName] = json::object();
-                    f["type"] = field.type;
-                    f["required"] = field.required;
-                    f["doc"] = stripIndentation(field.doc);
-                }
-            }
-        };
+        res["stores"] = std::move(stores);
+        res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
 
         return res.dump();
     }
@@ -458,7 +440,7 @@ void mainWrapped(int argc, char ** argv)
             if (!primOp->doc)
                 continue;
             b["args"] = primOp->args;
-            b["doc"] = trim(stripIndentation(*primOp->doc));
+            b["doc"] = trim(stripIndentation(primOp->doc->view()));
             if (primOp->experimentalFeature)
                 b["experimental-feature"] = primOp->experimentalFeature;
             builtinsJson.emplace(state.symbols[builtin.name], std::move(b));
@@ -467,7 +449,7 @@ void mainWrapped(int argc, char ** argv)
             auto b = nlohmann::json::object();
             if (!info.doc)
                 continue;
-            b["doc"] = trim(stripIndentation(info.doc));
+            b["doc"] = trim(stripIndentation(info.doc->view()));
             b["type"] = showType(info.type, false);
             if (info.impureOnly)
                 b["impure-only"] = true;

src/nix/nix-channel/nix-channel.cc

@@ -121,7 +121,7 @@ static void update(const StringSet & channelNames)
             // We want to download the url to a file to see if it's a tarball while also checking if we
             // got redirected in the process, so that we can grab the various parts of a nix channel
             // definition from a consistent location if the redirect changes mid-download.
-            auto result = fetchers::downloadFile(*store, fetchSettings, url, std::string(baseNameOf(url)));
+            auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url)));
             url = result.effectiveUrl;
 
             bool unpacked = false;
@@ -139,10 +139,10 @@ static void update(const StringSet & channelNames)
             if (!unpacked) {
                 // Download the channel tarball.
                 try {
-                    result = fetchers::downloadFile(*store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz");
+                    result = fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz");
                 } catch (FileTransferError & e) {
                     result =
-                        fetchers::downloadFile(*store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2");
+                        fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2");
                 }
             }
             // Regardless of where it came from, add the expression representing this channel to accumulated expression

src/nix/nix-env/user-env.cc

@@ -9,7 +9,7 @@
 #include "nix/expr/eval-inline.hh"
 #include "nix/store/profiles.hh"
 #include "nix/expr/print-ambiguous.hh"
-#include "nix/expr/static-string-data.hh"
+#include "nix/expr/string-data-static.hh"
 
 #include <limits>
 #include <sstream>

src/nix/prefetch.cc

@@ -138,7 +138,8 @@ std::tuple<StorePath, Hash> prefetchFile(
 
         Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url.to_string()));
 
-        auto info = store->addToStoreSlow(*name, makeFSSourceAccessor(tmpFile), method, hashAlgo, {}, expectedHash);
+        auto info = store->addToStoreSlow(
+            *name, PosixSourceAccessor::createAtRoot(tmpFile), method, hashAlgo, {}, expectedHash);
         storePath = info.path;
         assert(info.ca);
         hash = info.ca->hash;

src/nix/registry.cc

@@ -68,7 +68,7 @@ struct CmdRegistryList : StoreCommand
     {
         using namespace fetchers;
 
-        auto registries = getRegistries(fetchSettings, *store);
+        auto registries = getRegistries(fetchSettings, store);
 
         for (auto & registry : registries) {
             for (auto & entry : registry->entries) {
@@ -189,14 +189,14 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand
         auto registry = getRegistry();
         auto ref = parseFlakeRef(fetchSettings, url);
         auto lockedRef = parseFlakeRef(fetchSettings, locked);
-        auto resolvedInput = lockedRef.resolve(fetchSettings, *store).input;
-        auto resolved = resolvedInput.getAccessor(fetchSettings, *store).second;
+        registry->remove(ref.input);
+        auto resolvedInput = lockedRef.resolve(fetchSettings, store).input;
+        auto resolved = resolvedInput.getAccessor(fetchSettings, store).second;
         if (!resolved.isLocked(fetchSettings))
             warn("flake '%s' is not locked", resolved.to_string());
         fetchers::Attrs extraAttrs;
         if (ref.subdir != "")
             extraAttrs["dir"] = ref.subdir;
-        registry->remove(ref.input);
         registry->add(ref.input, resolved, extraAttrs);
         registry->write(getRegistryPath());
     }

tests/functional/flakes/flakes.sh

@@ -369,10 +369,6 @@ tar cfz "$TEST_ROOT"/flake.tar.gz -C "$TEST_ROOT" flake5
 
 nix build -o "$TEST_ROOT"/result file://"$TEST_ROOT"/flake.tar.gz
 
-nix flake clone "file://$TEST_ROOT/flake.tar.gz" --dest "$TEST_ROOT/unpacked"
-[[ -e $TEST_ROOT/unpacked/flake.nix ]]
-expectStderr 1 nix flake clone "file://$TEST_ROOT/flake.tar.gz" --dest "$TEST_ROOT/unpacked" | grep 'existing path'
-
 # Building with a tarball URL containing a SRI hash should also work.
 url=$(nix flake metadata --json file://"$TEST_ROOT"/flake.tar.gz | jq -r .url)
 [[ $url =~ sha256- ]]

tests/nixos/fetchers-substitute.nix

@@ -120,9 +120,7 @@
       path_info_json = substituter.succeed(f"nix path-info --json {tarball_store_path}").strip()
       path_info_dict = json.loads(path_info_json)
       # nix path-info returns a dict with store paths as keys
-      narHash_obj = path_info_dict[tarball_store_path]["narHash"]
-      # Convert from structured format {"algorithm": "sha256", "format": "base64", "hash": "..."} to SRI string
-      tarball_hash_sri = f"{narHash_obj['algorithm']}-{narHash_obj['hash']}"
+      tarball_hash_sri = path_info_dict[tarball_store_path]["narHash"]
       print(f"Tarball NAR hash (SRI): {tarball_hash_sri}")
 
       # Also get the old format hash for fetchTarball (which uses sha256 parameter)