Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f66a512290 | ||
|
|
8189f754e4 | ||
|
|
848ecbb5bc | ||
|
|
e2ac8756ae | ||
|
|
811645ed01 | ||
|
|
2a2b1ac083 | ||
|
|
300594f053 | ||
|
|
dbcde5f2e8 | ||
|
|
c57948e94a | ||
|
|
9dc0af1116 | ||
|
|
217768cf97 | ||
|
|
9f820e5a09 | ||
|
|
929152c0a8 | ||
|
|
c3a69d5c50 | ||
|
|
8fff7bafa7 |
No files matched your search
@@ -25,20 +25,5 @@ derivation {
|
||||
# Also don't bother substituting.
|
||||
allowSubstitutes = false;
|
||||
|
||||
__sandboxProfile = ''
|
||||
(allow sysctl-read)
|
||||
(allow file-read*
|
||||
(literal "/usr/lib/libSystem.dylib")
|
||||
(literal "/usr/lib/libSystem.B.dylib")
|
||||
(literal "/usr/lib/libobjc.A.dylib")
|
||||
(literal "/usr/lib/libobjc.dylib")
|
||||
(literal "/usr/lib/libauto.dylib")
|
||||
(literal "/usr/lib/libc++abi.dylib")
|
||||
(literal "/usr/lib/libc++.1.dylib")
|
||||
(literal "/usr/lib/libDiagnosticMessagesClient.dylib")
|
||||
(subpath "/usr/lib/system")
|
||||
(subpath "/dev"))
|
||||
'';
|
||||
|
||||
inherit chrootDeps;
|
||||
}
|
||||
@@ -280,8 +280,17 @@ with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixo
|
||||
|
||||
stdenv.mkDerivation { … }
|
||||
</programlisting>
|
||||
</para>
|
||||
|
||||
Note that when obtaining the hash with <varname>nix-prefetch-url
|
||||
<para>The fetched tarball is cached for a certain amount of time
|
||||
(1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>.
|
||||
You can change the cache timeout either on the command line with
|
||||
<option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or
|
||||
in the Nix configuration file with this option:
|
||||
<literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>.
|
||||
</para>
|
||||
|
||||
<para>Note that when obtaining the hash with <varname>nix-prefetch-url
|
||||
</varname> the option <varname>--unpack</varname> is required.
|
||||
</para>
|
||||
|
||||
@@ -420,7 +429,9 @@ stdenv.mkDerivation {
|
||||
<literal>"unknown"</literal> (for other kinds of files such as
|
||||
device nodes or fifos — but note that those cannot be copied to
|
||||
the Nix store, so if the predicate returns
|
||||
<literal>true</literal> for them, the copy will fail).</para>
|
||||
<literal>true</literal> for them, the copy will fail). If you
|
||||
exclude a directory, the entire corresponding subtree of
|
||||
<replaceable>e2</replaceable> will be excluded.</para>
|
||||
|
||||
</listitem>
|
||||
|
||||
@@ -468,8 +479,7 @@ builtins.fromJSON ''{"x": [1, 2, 3], "y": null}''
|
||||
</programlisting>
|
||||
|
||||
returns the value <literal>{ x = [ 1 2 3 ]; y = null;
|
||||
}</literal>. Floating point numbers are not
|
||||
supported.</para></listitem>
|
||||
}</literal>.</para></listitem>
|
||||
|
||||
</varlistentry>
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<section xmlns="http://docbook.org/ns/docbook"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
||||
version="5.0"
|
||||
xml:id="ssec-s3-substituter">
|
||||
|
||||
<title>Serving a Nix store via AWS S3 or S3-compatible Service</title>
|
||||
|
||||
<para>Nix has built-in support for storing and fetching store paths
|
||||
from Amazon S3 and S3 compatible services. This uses the same
|
||||
<emphasis>binary</emphasis> cache mechanism that Nix usually uses to
|
||||
fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para>
|
||||
|
||||
<para>In this example we will use the bucket named
|
||||
<literal>example-bucket</literal>.</para>
|
||||
|
||||
<section xml:id="ssec-s3-substituter-anonymous-reads">
|
||||
<title>Anonymous Reads to your S3-compatible binary cache</title>
|
||||
|
||||
<para>If your binary cache is publicly accessible and does not
|
||||
require authentication, the simplest and easiest way to use Nix with
|
||||
your S3 compatible binary cache is to use the HTTP URL for that
|
||||
cache.</para>
|
||||
|
||||
<para>For AWS S3 the binary cache URL for example bucket will be
|
||||
exactly <uri>https://example-bucket.s3.amazonaws.com</uri>. For S3
|
||||
compatible binary caches ago have to consult your software's
|
||||
documentation.</para>
|
||||
|
||||
<para>Your bucket will need the following bucket policy:</para>
|
||||
|
||||
<programlisting>
|
||||
<![CDATA[
|
||||
{
|
||||
"Id": "DirectReads",
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "AlowDirectReads",
|
||||
"Action": [
|
||||
"s3:GetObject"
|
||||
],
|
||||
"Effect": "Allow",
|
||||
"Resource": "arn:aws:s3:::example-bucket/*",
|
||||
"Principal": "*"
|
||||
}
|
||||
]
|
||||
}
|
||||
]]>
|
||||
</programlisting>
|
||||
</section>
|
||||
|
||||
<section xml:id="ssec-s3-substituter-authenticated-reads">
|
||||
<title>Authenticated Reads to your S3 binary cache</title>
|
||||
|
||||
<para>For AWS S3 the binary cache URL for example bucket will be
|
||||
exactly <uri>s3://example-bucket</uri>.</para>
|
||||
|
||||
<para>Nix will use the <link
|
||||
xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
|
||||
credential provider chain</link> for authenticating requests to
|
||||
Amazon S3.</para>
|
||||
|
||||
<para>Nix supports authenticated writes to S3 compatible binary
|
||||
caches but only supports Authenticated reads from Amazon S3.
|
||||
Additionally, the following limitations are in place for
|
||||
authenticated reads:</para>
|
||||
|
||||
<itemizedlist>
|
||||
<listitem><para>The bucket must actually be hosted by Amazon S3 and
|
||||
<emphasis>not</emphasis> an S3 compatible
|
||||
service.</para></listitem>
|
||||
|
||||
<listitem><para>The bucket must be within the
|
||||
<literal>us-east-1</literal> region.</para></listitem>
|
||||
|
||||
<listitem><para>The Amazon credentials, if stored in a credential
|
||||
profile, must be stored in the <literal>default</literal>
|
||||
profile.</para></listitem>
|
||||
</itemizedlist>
|
||||
|
||||
<para>Your bucket will need a bucket policy allowing the desired
|
||||
users to perform the <literal>s3:GetObject</literal> action on all
|
||||
objects in the bucket.</para>
|
||||
</section>
|
||||
|
||||
|
||||
<section xml:id="ssec-s3-substituter-authenticated-writes">
|
||||
<title>Authenticated Writes to your S3-compatible binary cache</title>
|
||||
|
||||
<para>Nix support fully supports writing to Amazon S3 and S3
|
||||
compatible buckets. The binary cache URL for our example bucket will
|
||||
be <uri>s3://example-bucket</uri>.</para>
|
||||
|
||||
<para>Nix will use the <link
|
||||
xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default
|
||||
credential provider chain</link> for authenticating requests to
|
||||
Amazon S3.</para>
|
||||
|
||||
<para>The following options can be specified as URL parameters to
|
||||
the S3 URL:</para>
|
||||
<variablelist>
|
||||
<varlistentry><term><literal>profile</literal></term>
|
||||
<listitem>
|
||||
<para>
|
||||
The name of the AWS configuration profile to use. By default
|
||||
Nix will use the <literal>default</literal> profile.
|
||||
</para>
|
||||
</listitem>
|
||||
</varlistentry>
|
||||
|
||||
<varlistentry><term><literal>region</literal></term>
|
||||
<listitem>
|
||||
<para>
|
||||
The region of the S3 bucket. <literal>us–east-1</literal> by
|
||||
default.
|
||||
</para>
|
||||
</listitem>
|
||||
</varlistentry>
|
||||
</variablelist>
|
||||
|
||||
<example><title>Uploading with non-default credential profile for Amazon S3</title>
|
||||
<para><command>nix copy --to ssh://machine nixpkgs.hello s3://example-bucket?profile=cache-upload</command></para>
|
||||
</example>
|
||||
|
||||
<para>The user writing to the bucket will need to perform the
|
||||
following actions against the bucket:</para>
|
||||
|
||||
<itemizedlist>
|
||||
<listitem><para><literal>s3:ListBucket</literal></para></listitem>
|
||||
<listitem><para><literal>s3:GetBucketLocation</literal></para></listitem>
|
||||
<listitem><para><literal>s3:ListObjects</literal></para></listitem>
|
||||
<listitem><para><literal>s3:GetObject</literal></para></listitem>
|
||||
<listitem><para><literal>s3:PutObject</literal></para></listitem>
|
||||
<listitem><para><literal>s3:ListBucketMultipartUploads</literal></para></listitem>
|
||||
<listitem><para><literal>s3:CreateMultipartUpload</literal></para></listitem>
|
||||
<listitem><para><literal>s3:ListMultipartUploadParts</literal></para></listitem>
|
||||
<listitem><para><literal>s3:AbortMultipartUpload</literal></para></listitem>
|
||||
</itemizedlist>
|
||||
</section>
|
||||
</section>
|
||||
@@ -15,5 +15,6 @@ packages between machines.</para>
|
||||
<xi:include href="binary-cache-substituter.xml" />
|
||||
<xi:include href="copy-closure.xml" />
|
||||
<xi:include href="ssh-substituter.xml" />
|
||||
<xi:include href="s3-substituter.xml" />
|
||||
|
||||
</chapter>
|
||||
+21
@@ -295,6 +295,24 @@ let
|
||||
'';
|
||||
|
||||
|
||||
installerScript =
|
||||
pkgs.runCommand "installer-script"
|
||||
{ buildInputs = [ build.x86_64-linux ];
|
||||
}
|
||||
''
|
||||
mkdir -p $out/nix-support
|
||||
|
||||
substitute ${./scripts/install.in} $out/install \
|
||||
${pkgs.lib.concatMapStrings
|
||||
(system: "--replace '@binaryTarball_${system}@' $(nix hash-file --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ")
|
||||
[ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
|
||||
} \
|
||||
--replace '@nixVersion@' ${build.x86_64-linux.src.version}
|
||||
|
||||
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
|
||||
# Aggregate job containing the release-critical jobs.
|
||||
release = pkgs.releaseTools.aggregate {
|
||||
name = "nix-${tarball.version}";
|
||||
@@ -304,14 +322,17 @@ let
|
||||
build.i686-linux
|
||||
build.x86_64-darwin
|
||||
build.x86_64-linux
|
||||
build.aarch64-linux
|
||||
binaryTarball.i686-linux
|
||||
binaryTarball.x86_64-darwin
|
||||
binaryTarball.x86_64-linux
|
||||
binaryTarball.aarch64-linux
|
||||
tests.remoteBuilds
|
||||
tests.nix-copy-closure
|
||||
tests.binaryTarball
|
||||
tests.evalNixpkgs
|
||||
tests.evalNixOS
|
||||
installerScript
|
||||
];
|
||||
};
|
||||
|
||||
|
||||
@@ -727,7 +727,7 @@ setup_default_profile() {
|
||||
_sudo "to installing a bootstrapping Nix in to the default Profile" \
|
||||
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX"
|
||||
|
||||
if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then
|
||||
if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then
|
||||
_sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \
|
||||
HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT"
|
||||
export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/bin/sh
|
||||
|
||||
# This script installs the Nix package manager on your system by
|
||||
# downloading a binary distribution and running its installer script
|
||||
# (which in turn creates and populates /nix).
|
||||
|
||||
{ # Prevent execution if this script was only partially downloaded
|
||||
oops() {
|
||||
echo "$0:" "$@" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \
|
||||
oops "Can\'t create temporary directory for downloading the Nix binary tarball")"
|
||||
cleanup() {
|
||||
rm -rf "$tmpDir"
|
||||
}
|
||||
trap cleanup EXIT INT QUIT TERM
|
||||
|
||||
require_util() {
|
||||
type "$1" > /dev/null 2>&1 || which "$1" > /dev/null 2>&1 ||
|
||||
oops "you do not have '$1' installed, which I need to $2"
|
||||
}
|
||||
|
||||
case "$(uname -s).$(uname -m)" in
|
||||
Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;;
|
||||
Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;;
|
||||
Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;;
|
||||
Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;;
|
||||
*) oops "sorry, there is no binary distribution of Nix for your platform";;
|
||||
esac
|
||||
|
||||
url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2"
|
||||
|
||||
tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")"
|
||||
|
||||
require_util curl "download the binary tarball"
|
||||
require_util bzcat "decompress the binary tarball"
|
||||
require_util tar "unpack the binary tarball"
|
||||
|
||||
echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..."
|
||||
curl -L "$url" -o "$tarball" || oops "failed to download '$url'"
|
||||
|
||||
if type sha256sum > /dev/null 2>&1; then
|
||||
hash2="$(sha256sum -b "$tarball" | cut -c1-64)"
|
||||
elif type shasum > /dev/null 2>&1; then
|
||||
hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)"
|
||||
elif type openssl > /dev/null 2>&1; then
|
||||
hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)"
|
||||
else
|
||||
oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'"
|
||||
fi
|
||||
|
||||
if [ "$hash" != "$hash2" ]; then
|
||||
oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2"
|
||||
fi
|
||||
|
||||
unpack=$tmpDir/unpack
|
||||
mkdir -p "$unpack"
|
||||
< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'"
|
||||
|
||||
script=$(echo "$unpack"/*/install)
|
||||
|
||||
[ -e "$script" ] || oops "installation script is missing from the binary tarball!"
|
||||
"$script" "$@"
|
||||
|
||||
} # End of wrapping
|
||||
@@ -1,5 +1,5 @@
|
||||
# Only execute this file once per shell.
|
||||
if [ -n "$__ETC_PROFILE_NIX_SOURCED" ]; then return; fi
|
||||
if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi
|
||||
__ETC_PROFILE_NIX_SOURCED=1
|
||||
|
||||
# Set up secure multi-user builds: non-root users build through the
|
||||
@@ -50,7 +50,7 @@ if test -w $HOME; then
|
||||
fi
|
||||
|
||||
# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work.
|
||||
if [ ! -z "$NIX_SSL_CERT_FILE" ]; then
|
||||
if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then
|
||||
: # Allow users to override the NIX_SSL_CERT_FILE
|
||||
elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch
|
||||
export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
@@ -220,6 +220,7 @@ struct CurlDownloader : public Downloader
|
||||
|
||||
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
|
||||
curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10);
|
||||
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
|
||||
curl_easy_setopt(req, CURLOPT_USERAGENT,
|
||||
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
|
||||
@@ -310,7 +311,7 @@ struct CurlDownloader : public Downloader
|
||||
// We treat most errors as transient, but won't retry when hopeless
|
||||
Error err = Transient;
|
||||
|
||||
if (httpStatus == 404 || code == CURLE_FILE_COULDNT_READ_FILE) {
|
||||
if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) {
|
||||
// The file is definitely not there
|
||||
err = NotFound;
|
||||
} else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) {
|
||||
@@ -340,6 +341,7 @@ struct CurlDownloader : public Downloader
|
||||
case CURLE_INTERFACE_FAILED:
|
||||
case CURLE_UNKNOWN_OPTION:
|
||||
case CURLE_SSL_CACERT_BADFILE:
|
||||
case CURLE_TOO_MANY_REDIRECTS:
|
||||
err = Misc;
|
||||
break;
|
||||
default: // Shut up warnings
|
||||
@@ -725,8 +727,13 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
|
||||
storePath = unpackedStorePath;
|
||||
}
|
||||
|
||||
if (expectedStorePath != "" && storePath != expectedStorePath)
|
||||
throw nix::Error("store path mismatch in file downloaded from '%s'", url);
|
||||
if (expectedStorePath != "" && storePath != expectedStorePath) {
|
||||
Hash gotHash = unpack
|
||||
? hashPath(expectedHash.type, store->toRealPath(storePath)).first
|
||||
: hashFile(expectedHash.type, store->toRealPath(storePath));
|
||||
throw nix::Error("hash mismatch in file downloaded from '%s': expected hash '%s', got '%s'",
|
||||
url, expectedHash.to_string(), gotHash.to_string());
|
||||
}
|
||||
|
||||
return store->toRealPath(storePath);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user