Compare commits

..
15 Commits
Author SHA1 Message Date
Graham Christensen f66a512290 Document s3 substitutions
(cherry picked from commit b27431b7cb)
2018-08-02 08:24:47 -04:00
Eelco Dolstra 8189f754e4 Merge pull request #2313 from LnL7/no-sandbox-profile
buildenv: remove darwin sandbox profile
2018-07-31 10:34:34 +02:00
Daiderd Jordan 848ecbb5bc buildenv: remove sandbox profile
This shouldn't be necessary anymore and it breaks nix-env/nix-channel
when non 'relaxed' sandboxing is enabled.
2018-07-28 00:32:29 +02:00
Eelco Dolstra e2ac8756ae Handle HTTP 410 Gone errors
(cherry picked from commit 691b7582c7)
2018-06-20 16:14:20 +02:00
Eelco Dolstra 811645ed01 Detect redirect loops
Fixes #2225.

(cherry picked from commit b920b90857)
2018-06-20 16:13:30 +02:00
Will Dietz 2a2b1ac083 download: improve error for hash mismatch ("store mismatch")
Fixes #1905

(cherry picked from commit 74da813912)
2018-06-20 16:13:15 +02:00
Shea Levy 300594f053 Merge pull request #2221 from 7c6f434c/doc-floats-2.0
backport: manual: builtins.fromJSON: floats
2018-06-10 11:59:20 -04:00
Michael Raskin dbcde5f2e8 manual: builtins.fromJSON: remove the claim that floats are not allowed
floating-point numbers are supported now, including the fromJSON
builtin. Reported on IRC by inquisitiv3

(cherry picked from commit 17bc757980)
2018-06-10 16:40:53 +02:00
Shea Levy c57948e94a Explicitly describe the effects of filterSource excluding a directory.
(cherry picked from commit bbbfc180d9)
2018-05-30 16:32:31 -04:00
Eelco Dolstra 9dc0af1116 release.nix: Generate the installer script
(cherry picked from commit 7c3c635d4f)
2018-05-30 17:45:03 +02:00
Eelco Dolstra 217768cf97 Move installer script from nixos-homepage
(cherry picked from commit 4caaa4c5fe)
2018-05-30 17:45:02 +02:00
Graham Christensen 9f820e5a09 install-multi-user: support 'set -u' runs, closes #2193
(cherry picked from commit 6ba1726eeb)
2018-05-30 17:44:42 +02:00
Graham Christensen 929152c0a8 nix-daemon.sh profile script: operate under set -u
If the profile is sourced inside a script with `set -u`, the check for
__ETC_PROFILE_NIX_SOURCED and NIX_SSL_CERT_FILE would raise an error.
A simple guard around this check allows the script to operate under
standard environments (where it is fairly reasonable to assume USER
and HOME are set.)

(cherry picked from commit d459d3307c)
2018-05-30 17:44:42 +02:00
Michael Mercier c3a69d5c50 Explain fetchTarball timeout behavior in the doc
(cherry picked from commit 8dd2e28374)
2018-05-30 17:44:22 +02:00
Eelco Dolstra 8fff7bafa7 Bump version 2018-05-30 17:43:16 +02:00
10 changed files with 259 additions and 26 deletions

No files matched your search

-15
View File
@@ -25,20 +25,5 @@ derivation {
# Also don't bother substituting.
allowSubstitutes = false;
__sandboxProfile = ''
(allow sysctl-read)
(allow file-read*
(literal "/usr/lib/libSystem.dylib")
(literal "/usr/lib/libSystem.B.dylib")
(literal "/usr/lib/libobjc.A.dylib")
(literal "/usr/lib/libobjc.dylib")
(literal "/usr/lib/libauto.dylib")
(literal "/usr/lib/libc++abi.dylib")
(literal "/usr/lib/libc++.1.dylib")
(literal "/usr/lib/libDiagnosticMessagesClient.dylib")
(subpath "/usr/lib/system")
(subpath "/dev"))
'';
inherit chrootDeps;
}
+14 -4
View File
@@ -280,8 +280,17 @@ with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixo
stdenv.mkDerivation { … }
</programlisting>
</para>
Note that when obtaining the hash with <varname>nix-prefetch-url
<para>The fetched tarball is cached for a certain amount of time
(1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>.
You can change the cache timeout either on the command line with
<option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or
in the Nix configuration file with this option:
<literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>.
</para>
<para>Note that when obtaining the hash with <varname>nix-prefetch-url
</varname> the option <varname>--unpack</varname> is required.
</para>
@@ -420,7 +429,9 @@ stdenv.mkDerivation {
<literal>"unknown"</literal> (for other kinds of files such as
device nodes or fifos — but note that those cannot be copied to
the Nix store, so if the predicate returns
<literal>true</literal> for them, the copy will fail).</para>
<literal>true</literal> for them, the copy will fail). If you
exclude a directory, the entire corresponding subtree of
<replaceable>e2</replaceable> will be excluded.</para>
</listitem>
@@ -468,8 +479,7 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
</programlisting>
returns the value <literal>{ x = [ 1 2 3 ]; y = null;
}</literal>. Floating point numbers are not
supported.</para></listitem>
}</literal>.</para></listitem>
</varlistentry>
+142
View File
@@ -0,0 +1,142 @@
<?xml version="1.0" encoding="utf-8"?>
<section xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
version="5.0"
xml:id="ssec-s3-substituter">
<title>Serving a Nix store via AWS S3 or S3-compatible Service</title>
<para>Nix has built-in support for storing and fetching store paths
from Amazon S3 and S3 compatible services. This uses the same
<emphasis>binary</emphasis> cache mechanism that Nix usually uses to
fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
<para>In this example we will use the bucket named
<literal>example-bucket</literal>.</para>
<section xml:id="ssec-s3-substituter-anonymous-reads">
<title>Anonymous Reads to your S3-compatible binary cache</title>
<para>If your binary cache is publicly accessible and does not
require authentication, the simplest and easiest way to use Nix with
your S3 compatible binary cache is to use the HTTP URL for that
cache.</para>
<para>For AWS S3 the binary cache URL for example bucket will be
exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3
compatible binary caches ago have to consult your software's
documentation.</para>
<para>Your bucket will need the following bucket policy:</para>
<programlisting>
<![CDATA[
{
"Id": "DirectReads",
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AlowDirectReads",
"Action": [
"s3:GetObject"
],
"Effect": "Allow",
"Resource": "arn:aws:s3:::example-bucket/*",
"Principal": "*"
}
]
}
]]>
</programlisting>
</section>
<section xml:id="ssec-s3-substituter-authenticated-reads">
<title>Authenticated Reads to your S3 binary cache</title>
<para>For AWS S3 the binary cache URL for example bucket will be
exactly <uri>s3://example-bucket</uri>.</para>
<para>Nix will use the <link
xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
credential provider chain</link> for authenticating requests to
Amazon S3.</para>
<para>Nix supports authenticated writes to S3 compatible binary
caches but only supports Authenticated reads from Amazon S3.
Additionally, the following limitations are in place for
authenticated reads:</para>
<itemizedlist>
<listitem><para>The bucket must actually be hosted by Amazon S3 and
<emphasis>not</emphasis> an S3 compatible
service.</para></listitem>
<listitem><para>The bucket must be within the
<literal>us-east-1</literal> region.</para></listitem>
<listitem><para>The Amazon credentials, if stored in a credential
profile, must be stored in the <literal>default</literal>
profile.</para></listitem>
</itemizedlist>
<para>Your bucket will need a bucket policy allowing the desired
users to perform the <literal>s3:GetObject</literal> action on all
objects in the bucket.</para>
</section>
<section xml:id="ssec-s3-substituter-authenticated-writes">
<title>Authenticated Writes to your S3-compatible binary cache</title>
<para>Nix support fully supports writing to Amazon S3 and S3
compatible buckets. The binary cache URL for our example bucket will
be <uri>s3://example-bucket</uri>.</para>
<para>Nix will use the <link
xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
credential provider chain</link> for authenticating requests to
Amazon S3.</para>
<para>The following options can be specified as URL parameters to
the S3 URL:</para>
<variablelist>
<varlistentry><term><literal>profile</literal></term>
<listitem>
<para>
The name of the AWS configuration profile to use. By default
Nix will use the <literal>default</literal> profile.
</para>
</listitem>
</varlistentry>
<varlistentry><term><literal>region</literal></term>
<listitem>
<para>
The region of the S3 bucket. <literal>us–east-1</literal> by
default.
</para>
</listitem>
</varlistentry>
</variablelist>
<example><title>Uploading with non-default credential profile for Amazon S3</title>
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
</example>
<para>The user writing to the bucket will need to perform the
following actions against the bucket:</para>
<itemizedlist>
<listitem><para><literal>s3:ListBucket</literal></para></listitem>
<listitem><para><literal>s3:GetBucketLocation</literal></para></listitem>
<listitem><para><literal>s3:ListObjects</literal></para></listitem>
<listitem><para><literal>s3:GetObject</literal></para></listitem>
<listitem><para><literal>s3:PutObject</literal></para></listitem>
<listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem>
<listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem>
<listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem>
<listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem>
</itemizedlist>
</section>
</section>
+1
View File
@@ -15,5 +15,6 @@ packages between machines.</para>
<xi:include href="binary-cache-substituter.xml" />
<xi:include href="copy-closure.xml" />
<xi:include href="ssh-substituter.xml" />
<xi:include href="s3-substituter.xml" />
</chapter>
+21
View File
@@ -295,6 +295,24 @@ let
'';
installerScript =
pkgs.runCommand "installer-script"
{ buildInputs = [ build.x86_64-linux ];
}
''
mkdir -p $out/nix-support
substitute ${./scripts/install.in} $out/install \
${pkgs.lib.concatMapStrings
(system: "--replace '@binaryTarball_${system}@' $(nix hash-file --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
[ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
} \
--replace '@nixVersion@' ${build.x86_64-linux.src.version}
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
'';
# Aggregate job containing the release-critical jobs.
release = pkgs.releaseTools.aggregate {
name = "nix-${tarball.version}";
@@ -304,14 +322,17 @@ let
build.i686-linux
build.x86_64-darwin
build.x86_64-linux
build.aarch64-linux
binaryTarball.i686-linux
binaryTarball.x86_64-darwin
binaryTarball.x86_64-linux
binaryTarball.aarch64-linux
tests.remoteBuilds
tests.nix-copy-closure
tests.binaryTarball
tests.evalNixpkgs
tests.evalNixOS
installerScript
];
};
+1 -1
View File
@@ -727,7 +727,7 @@ setup_default_profile() {
_sudo "to installing a bootstrapping Nix in to the default Profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then
_sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
+67
View File
@@ -0,0 +1,67 @@
#!/bin/sh
# This script installs the Nix package manager on your system by
# downloading a binary distribution and running its installer script
# (which in turn creates and populates /nix).
{ # Prevent execution if this script was only partially downloaded
oops() {
echo "$0:" "$@" >&2
exit 1
}
tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
oops "Can\'t create temporary directory for downloading the Nix binary tarball")"
cleanup() {
rm -rf "$tmpDir"
}
trap cleanup EXIT INT QUIT TERM
require_util() {
type "$1" > /dev/null 2>&1 || which "$1" > /dev/null 2>&1 ||
oops "you do not have '$1' installed, which I need to $2"
}
case "$(uname -s).$(uname -m)" in
Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;;
Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
*) oops "sorry, there is no binary distribution of Nix for your platform";;
esac
url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2"
tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")"
require_util curl "download the binary tarball"
require_util bzcat "decompress the binary tarball"
require_util tar "unpack the binary tarball"
echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
if type sha256sum > /dev/null 2>&1; then
hash2="$(sha256sum -b "$tarball" | cut -c1-64)"
elif type shasum > /dev/null 2>&1; then
hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)"
elif type openssl > /dev/null 2>&1; then
hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)"
else
oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'"
fi
if [ "$hash" != "$hash2" ]; then
oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2"
fi
unpack=$tmpDir/unpack
mkdir -p "$unpack"
< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'"
script=$(echo "$unpack"/*/install)
[ -e "$script" ] || oops "installation script is missing from the binary tarball!"
"$script" "$@"
} # End of wrapping
+2 -2
View File
@@ -1,5 +1,5 @@
# Only execute this file once per shell.
if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
__ETC_PROFILE_NIX_SOURCED=1
# Set up secure multi-user builds: non-root users build through the
@@ -50,7 +50,7 @@ if test -w $HOME; then
fi
# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
if [ ! -z "$NIX_SSL_CERT_FILE" ]; then
if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
: # Allow users to override the NIX_SSL_CERT_FILE
elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
+10 -3
View File
@@ -220,6 +220,7 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10);
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
@@ -310,7 +311,7 @@ struct CurlDownloader : public Downloader
// We treat most errors as transient, but won't retry when hopeless
Error err = Transient;
if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) {
if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
// The file is definitely not there
err = NotFound;
} else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
@@ -340,6 +341,7 @@ struct CurlDownloader : public Downloader
case CURLE_INTERFACE_FAILED:
case CURLE_UNKNOWN_OPTION:
case CURLE_SSL_CACERT_BADFILE:
case CURLE_TOO_MANY_REDIRECTS:
err = Misc;
break;
default: // Shut up warnings
@@ -725,8 +727,13 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
storePath = unpackedStorePath;
}
if (expectedStorePath != "" && storePath != expectedStorePath)
throw nix::Error("store path mismatch in file downloaded from '%s'", url);
if (expectedStorePath != "" && storePath != expectedStorePath) {
Hash gotHash = unpack
? hashPath(expectedHash.type, store->toRealPath(storePath)).first
: hashFile(expectedHash.type, store->toRealPath(storePath));
throw nix::Error("hash mismatch in file downloaded from '%s': expected hash '%s', got '%s'",
url, expectedHash.to_string(), gotHash.to_string());
}
return store->toRealPath(storePath);
}
+1 -1
View File
@@ -1 +1 @@
2.0.3
2.0.4