Compare commits

..
4 Commits
Author SHA1 Message Date
Eelco Dolstra 3e95bb49be * Move this back temporarily. 2004-11-14 15:04:51 +00:00
Eelco Dolstra 84cc89b585 * Tag release 0.6. 2004-11-14 12:58:20 +00:00
Eelco Dolstra ac76ee5088 * Mark stable release. 2004-11-14 12:09:31 +00:00
Eelco Dolstra f8e4f731d4 * Release branch for Nix 0.6. 2004-11-14 12:04:21 +00:00
161 changed files with 4693 additions and 10994 deletions

No files matched your search

+1 -10
View File
@@ -1,6 +1,5 @@
SUBDIRS = externals src scripts corepkgs doc misc tests
EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh \
svn-revision nix.conf.example
EXTRA_DIST = substitute.mk nix.spec nix.spec.in bootstrap.sh svn-revision
include ./substitute.mk
@@ -13,11 +12,6 @@ relname:
echo -n $(distdir) > relname
install-data-local: init-state
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix
if ! test -e $(DESTDIR)$(sysconfdir)/nix/nix.conf; then \
$(INSTALL_DATA) nix.conf.example $(DESTDIR)$(sysconfdir)/nix/nix.conf; \
fi
if INIT_STATE
if SETUID_HACK
@@ -28,16 +22,13 @@ init-state:
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/db
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/log/nix/drvs
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/profiles
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/gcroots
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(prefix)/store
$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
# $(bindir)/nix-store --init
else
init-state:
+2 -198
View File
@@ -1,200 +1,4 @@
Release 0.8.1 (April 13, 2005)
This is a bug fix release.
* Patch downloading was broken.
* The garbage collector would not delete paths that had references
from invalid (but substitutable) paths.
Release 0.8 (April 11, 2005)
NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a
result, `nix-pull' manifests and channels built for Nix 0.7 and below
will now work anymore. However, the Nix expression language has not
changed, so you can still build from source. Also, existing user
environments continue to work. Nix 0.8 will automatically upgrade the
database schema of previous installations when it is first run.
If you get the error message
you have an old-style manifest `/nix/var/nix/manifests/[...]';
please delete it
you should delete previously downloaded manifests:
$ rm /nix/var/nix/manifests/*
If `nix-channel' gives the error message
manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST'
is too old (i.e., for Nix <= 0.7)
then you should unsubscribe from the offending channel (`nix-channel
--remove URL'; leave out `/MANIFEST'), and subscribe to the same URL,
with `channels' replaced by `channels-v3' (e.g.,
http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable).
Nix 0.8 has the following improvements:
* The cryptographic hashes used in store paths are now 160 bits long,
but encoded in base-32 so that they are still only 32 characters
long (e.g., /nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0).
(This is actually a 160 bit truncation of a SHA-256 hash.)
* Big cleanups and simplifications of the basic store semantics. The
notion of "closure store expressions" is gone (and so is the notion
of "successors"); the file system references of a store path are now
just stored in the database.
For instance, given any store path, you can query its closure:
$ nix-store -qR $(which firefox)
... lots of paths ...
Also, Nix now remembers for each store path the derivation that
built it (the "deriver"):
$ nix-store -qR $(which firefox)
/nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv
So to see the build-time dependencies, you can do
$ nix-store -qR $(nix-store -qd $(which firefox))
or, in a nicer format:
$ nix-store -q --tree $(nix-store -qd $(which firefox))
File system references are also stored in reverse. For instance,
you can query all paths that directly or indirectly use a certain
Glibc:
$ nix-store -q --referers-closure \
/nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4
* The concept of fixed-output derivations has been formalised.
Previously, functions such as `fetchurl' in Nixpkgs used a hack
(namely, explicitly specifying a store path hash) to prevent changes
to, say, the URL of the file from propagating upwards through the
dependency graph, causing rebuilds of everything. This can now be
done cleanly by specifying the `outputHash' and `outputHashAlgo'
attributes. Nix itself checks that the content of the output has
the specified hash. (This is important for maintaining certain
invariants necessary for future work on secure shared stores.)
* One-click installation :-) It is now possible to install any
top-level component in Nixpkgs directly, through the web - see,
e.g., http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/. All you
have to do is associate `/nix/bin/nix-install-package' with the MIME
type `application/nix-package' (or the extension `.nixpkg'), and
clicking on a package link will cause it to be installed, with all
appropriate dependencies. If you just want to install some specific
application, this is easier than subscribing to a channel.
* `nix-store -r PATHS' now builds all the derivations PATHS in
parallel. Previously it did them sequentially (though exploiting
possible parallelism between subderivations). This is nice for
build farms.
* `nix-channel' has new operations `--list' and `--remove'.
* New ways of installing components into user environments:
- Copy from another user environment:
$ nix-env -i --from-profile .../other-profile firefox
- Install a store derivation directly (bypassing the Nix expression
language entirely):
$ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv
(This is used to implement `nix-install-package', which is
therefore immune to evolution in the Nix expression language.)
- Install an already built store path directly:
$ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1
- Install the result of a Nix expression specified as a command-line
argument:
$ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'
The difference with the normal installation mode is that `-E' does
not use the `name' attributes of derivations. Therefore, this can
be used to disambiguate multiple derivations with the same name.
* A hash of the contents of a store path is now stored in the database
after a succesful build. This allows you to check whether store
paths have been tampered with: `nix-store --verify --check-contents'.
* Implemented a concurrent garbage collector. It is now always safe
to run the garbage collector, even if other Nix operations are
happening simultaneously.
However, there can still be GC races if you use `nix-instantiate'
and `nix-store -r' directly to build things. To prevent races, use
the `--add-root' flag of those commands.
* The garbage collector now finally deletes paths in the right order
(i.e., topologically sorted under the `references' relation), thus
making it safe to interrupt the collector without risking a store
that violates the closure invariant.
* Likewise, the substitute mechanism now downloads files in the right
order, thus preserving the closure invariant at all times.
* The result of `nix-build' is now registered as a root of the garbage
collector. If the `./result' link is deleted, the GC root
disappears automatically.
* The behaviour of the garbage collector can be changed globally by
setting options in `/nix/etc/nix/nix.conf'.
- `gc-keep-derivations' specifies whether deriver links should be
followed when searching for live paths.
- `gc-keep-outputs' specifies whether outputs of derivations should
be followed when searching for live paths.
- `env-keep-derivations' specifies whether user environments should
store the paths of derivations when they are added (thus keeping
the derivations alive).
* New `nix-env' query flags `--drv-path' and `--out-path'.
* `fetchurl' allows SHA-1 and SHA-256 in addition to MD5. Just
specify the attribute `sha1' or `sha256' instead of `md5'.
* Manual updates.
Release 0.7 (January 12, 2005)
* Binary patching. When upgrading components using pre-built binaries
(through nix-pull / nix-channel), Nix can automatically download and
apply binary patches to already installed components instead of full
downloads. Patching is "smart": if there is a *sequence* of patches
to an installed component, Nix will use it. Patches are currently
generated automatically between Nixpkgs (pre-)releases.
* Simplifications to the substitute mechanism.
* Nix-pull now stores downloaded manifests in /nix/var/nix/manifests.
* Metadata on files in the Nix store is canonicalised after builds:
the last-modified timestamp is set to 0 (00:00:00 1/1/1970), the
mode is set to 0444 or 0555 (readable and possibly executable by
all; setuid/setgid bits are dropped), and the group is set to the
default. This ensures that the result of a build and an
installation through a substitute is the same; and that timestamp
dependencies are revealed.
Release 0.6 (November 14, 2004)
Version 0.6
Major changes include the following:
@@ -260,6 +64,6 @@ Major changes include the following:
* Many bug fixes.
Release 0.5 and earlier
Version 0.5 and earlier
Please refer to the Subversion commit log messages.
+2 -6
View File
@@ -1,9 +1,5 @@
*** Nix ***
For installation and usage instructions, please read the manual, which
can be found in `docs/manual/manual.html', and additionally at the Nix
website at <http://www.cs.uu.nl/groups/ST/Trace/Nix>.
Acknowledgments
This product includes software developed by the OpenSSL Project for
use in the OpenSSL Toolkit (http://www.OpenSSL.org/)
-252
View File
@@ -1,252 +0,0 @@
#! /usr/bin/perl -w -I /home/eelco/.nix-profile/lib/site_perl
use strict;
use XML::LibXML;
#use XML::Simple;
my $blacklistFN = shift @ARGV;
die unless defined $blacklistFN;
my $userEnv = shift @ARGV;
die unless defined $userEnv;
# Read the blacklist.
my $parser = XML::LibXML->new();
my $blacklist = $parser->parse_file($blacklistFN)->getDocumentElement;
#print $blacklist->toString() , "\n";
# Get all the elements of the user environment.
my $userEnvElems = `nix-store --query --references '$userEnv'`;
die "cannot query user environment elements" if $? != 0;
my @userEnvElems = split ' ', $userEnvElems;
my %storePathHashes;
sub getElemNodes {
my $node = shift;
my @elems = ();
foreach my $node ($node->getChildNodes) {
push @elems, $node if $node->nodeType == XML_ELEMENT_NODE;
}
return @elems;
}
my %referencesCache;
sub getReferences {
my $path = shift;
return $referencesCache{$path} if defined $referencesCache{$path};
my $references = `nix-store --query --references '$path'`;
die "cannot query references" if $? != 0;
$referencesCache{$path} = [split ' ', $references];
return $referencesCache{$path};
}
my %attrsCache;
sub getAttr {
my $path = shift;
my $name = shift;
my $key = "$path/$name";
return $referencesCache{$key} if defined $referencesCache{$key};
my $value = `nix-store --query --binding '$name' '$path' 2> /dev/null`;
$value = "" if $? != 0; # !!!
chomp $value;
$referencesCache{$key} = $value;
return $value;
}
sub evalCondition;
sub traverse {
my $done = shift;
my $set = shift;
my $path = shift;
my $stopCondition = shift;
return if defined $done->{$path};
$done->{$path} = 1;
$set->{$path} = 1;
# print " in $path\n";
if (!evalCondition({$path => 1}, $stopCondition)) {
# print " STOPPING in $path\n";
return;
}
# Get the requisites of the deriver.
foreach my $reference (@{getReferences $path}) {
traverse($done, $set, $reference, $stopCondition);
}
}
sub evalSet {
my $inSet = shift;
my $expr = shift;
my $name = $expr->getName;
if ($name eq "traverse") {
my $stopCondition = (getElemNodes $expr)[0];
my $done = { };
my $set = { };
foreach my $path (keys %{$inSet}) {
traverse($done, $set, $path, $stopCondition);
}
return $set;
}
else {
die "unknown element `$name'";
}
}
# Function for evaluating conditions.
sub evalCondition {
my $storePaths = shift;
my $condition = shift;
my $elemName = $condition->getName;
if ($elemName eq "containsSource") {
my $hash = $condition->attributes->getNamedItem("hash")->getValue;
foreach my $path (keys %{$storePathHashes{$hash}}) {
return 1 if defined $storePaths->{$path};
}
return 0;
}
elsif ($elemName eq "hasName") {
my $nameRE = $condition->attributes->getNamedItem("name")->getValue;
foreach my $path (keys %{$storePaths}) {
return 1 if $path =~ /$nameRE/;
}
return 0;
}
elsif ($elemName eq "hasAttr") {
my $name = $condition->attributes->getNamedItem("name")->getValue;
my $valueRE = $condition->attributes->getNamedItem("value")->getValue;
foreach my $path (keys %{$storePaths}) {
if ($path =~ /\.drv$/) {
my $value = getAttr($path, $name);
# print " $path $name $value\n";
return 1 if $value =~ /$valueRE/;
}
}
return 0;
}
elsif ($elemName eq "and") {
my $result = 1;
foreach my $node (getElemNodes $condition) {
$result &= evalCondition($storePaths, $node);
}
return $result;
}
elsif ($elemName eq "not") {
return !evalCondition($storePaths, (getElemNodes $condition)[0]);
}
elsif ($elemName eq "within") {
my @elems = getElemNodes $condition;
my $set = evalSet($storePaths, $elems[0]);
return evalCondition($set, $elems[1]);
}
elsif ($elemName eq "true") {
return 1;
}
elsif ($elemName eq "false") {
return 0;
}
else {
die "unknown element `$elemName'";
}
}
sub evalOr {
my $storePaths = shift;
my $nodes = shift;
my $result = 0;
foreach my $node (@{$nodes}) {
$result |= evalCondition($storePaths, $node);
}
return $result;
}
# Iterate over all elements, check them.
foreach my $userEnvElem (@userEnvElems) {
# Get the deriver of this path.
my $deriver = `nix-store --query --deriver '$userEnvElem'`;
die "cannot query deriver" if $? != 0;
chomp $deriver;
if ($deriver eq "unknown-deriver") {
# print " deriver unknown, cannot check sources\n";
next;
}
print "CHECKING $userEnvElem\n";
# Get the requisites of the deriver.
# my $requisites = `nix-store --query --requisites --include-outputs '$deriver'`;
# die "cannot query requisites" if $? != 0;
# my @requisites = split ' ', $requisites;
# Get the hashes of the requisites.
# my $hashes = `nix-store --query --hash @requisites`;
# die "cannot query hashes" if $? != 0;
# my @hashes = split ' ', $hashes;
# for (my $i = 0; $i < scalar @requisites; $i++) {
# die unless $i < scalar @hashes;
# my $hash = $hashes[$i];
# $storePathHashes{$hash} = {} unless defined $storePathHashes{$hash};
# my $r = $storePathHashes{$hash}; # !!! fix
# $$r{$requisites[$i]} = 1;
# }
# Evaluate each blacklist item.
foreach my $item ($blacklist->getChildrenByTagName("item")) {
my $itemId = $item->getAttributeNode("id")->getValue;
# print " CHECKING FOR $itemId\n";
my $condition = ($item->getChildrenByTagName("condition"))[0];
die unless $condition;
# Evaluate the condition.
my @elems = getElemNodes $condition;
if (evalOr({$deriver => 1}, \@elems)) {
# Oops, condition triggered.
my $reason = ($item->getChildrenByTagName("reason"))[0]->getChildNodes->to_literal;
$reason =~ s/\s+/ /g;
$reason =~ s/^\s+//g;
print " VULNERABLE TO `$itemId': $reason\n";
}
}
}
+8 -9
View File
@@ -1,4 +1,4 @@
AC_INIT(nix, "0.8.1")
AC_INIT(nix, "0.6")
AC_CONFIG_SRCDIR(README)
AC_CONFIG_AUX_DIR(config)
AM_INIT_AUTOMAKE
@@ -81,19 +81,17 @@ AC_PATH_PROG(xsltproc, xsltproc, false)
AC_PATH_PROG(flex, flex, false)
AC_PATH_PROG(bison, bison, false)
NEED_PROG(perl, perl)
NEED_PROG(tar, tar)
NEED_PROG(cat, cat)
AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH],
[path of cat, mkdir, etc.]),
coreutils=$withval, coreutils=$(dirname $cat))
AC_SUBST(coreutils)
AC_ARG_WITH(docbook-catalog, AC_HELP_STRING([--with-docbook-catalog=PATH],
[path of the DocBook XML DTD]),
docbookcatalog=$withval, docbookcatalog=/docbook-dtd-missing)
AC_SUBST(docbookcatalog)
AC_ARG_WITH(docbook-ebnf-catalog, AC_HELP_STRING([--with-docbook-ebnf-catalog=PATH],
[path of the DocBook XML EBNF module DTD]),
docbookebnfcatalog=$withval, docbookcatalog=/docbook-ebnf-dtd-missing)
AC_SUBST(docbookebnfcatalog)
AC_ARG_WITH(docbook-xsl, AC_HELP_STRING([--with-docbook-xsl=PATH],
[path of the DocBook XSL stylesheets]),
docbookxsl=$withval, docbookxsl=/docbook-xsl-missing)
@@ -191,12 +189,13 @@ AC_CONFIG_FILES([Makefile
src/nix-instantiate/Makefile
src/nix-env/Makefile
src/log2xml/Makefile
src/bsdiff-4.2/Makefile
scripts/Makefile
corepkgs/Makefile
corepkgs/fetchurl/Makefile
corepkgs/nar/Makefile
corepkgs/buildenv/Makefile
corepkgs/channels/Makefile
corepkgs/nix-pull/Makefile
doc/Makefile
doc/manual/Makefile
misc/Makefile
+1 -1
View File
@@ -1 +1 @@
SUBDIRS = nar buildenv channels
SUBDIRS = fetchurl nar buildenv channels nix-pull
+1 -15
View File
@@ -25,7 +25,6 @@ sub createLinks {
if ($srcFile =~ /\/propagated-build-inputs$/ ||
$srcFile =~ /\/nix-support$/ ||
$srcFile =~ /\/perllocal.pod$/ ||
$srcFile =~ /\/log$/)
{
# Do nothing.
@@ -73,27 +72,13 @@ sub createLinks {
my %done;
sub addPkg;
sub addPkg {
my $pkgDir = shift;
return if (defined $done{$pkgDir});
$done{$pkgDir} = 1;
print "adding $pkgDir\n";
createLinks("$pkgDir", "$out");
my $propagatedFN = "$pkgDir/nix-support/propagated-build-inputs";
if (-e $propagatedFN) {
open PROP, "<$propagatedFN" or die;
my $propagated = <PROP>;
close PROP;
my @propagated = split ' ', $propagated;
foreach my $p (@propagated) {
addPkg $p;
}
}
}
@@ -101,6 +86,7 @@ my @args = split ' ', $ENV{"derivations"};
while (scalar @args > 0) {
my $drvPath = shift @args;
print "adding $drvPath\n";
addPkg($drvPath);
}
+7 -5
View File
@@ -1,7 +1,9 @@
#! @shell@ -e
@coreutils@/mkdir $out
@coreutils@/mkdir $out/tmp
export PATH=/bin:/usr/bin # !!! impure
mkdir $out
mkdir $out/tmp
cd $out/tmp
expr=$out/default.nix
@@ -10,8 +12,8 @@ echo '[' > $expr
nr=0
for i in $inputs; do
echo "unpacking $i"
@bunzip2@ < $i | @tar@ xvf -
@coreutils@/mv * ../$nr # !!! hacky
@bunzip2@ < $i | tar xvf -
mv * ../$nr # !!! hacky
echo "(import ./$nr)" >> $expr
nr=$(($nr + 1))
done
@@ -19,4 +21,4 @@ done
echo ']' >> $expr
cd ..
@coreutils@/rmdir tmp
rmdir tmp
+11
View File
@@ -0,0 +1,11 @@
all-local: builder.sh
install-exec-local:
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/fetchurl
include ../../substitute.mk
EXTRA_DIST = default.nix builder.sh.in
+19
View File
@@ -0,0 +1,19 @@
#! @shell@ -e
export PATH=/bin:/usr/bin
echo "downloading $url into $out"
prefetch=@storedir@/nix-prefetch-url-$md5
if test -f "$prefetch"; then
echo "using prefetched $prefetch";
mv $prefetch $out
else
@curl@ --fail --location --max-redirs 20 "$url" > "$out"
fi
actual=$(@bindir@/nix-hash --flat $out)
if test "$actual" != "$md5"; then
echo "hash is $actual, expected $md5"
exit 1
fi
+8
View File
@@ -0,0 +1,8 @@
{system, url, md5}:
derivation {
name = baseNameOf (toString url);
builder = ./builder.sh;
id = md5;
inherit system url md5;
}
+4 -2
View File
@@ -1,11 +1,13 @@
all-local: nar.sh
all-local: nar.sh unnar.sh
install-exec-local:
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nar
$(INSTALL_DATA) nar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
$(INSTALL_PROGRAM) nar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
$(INSTALL_DATA) unnar.nix $(DESTDIR)$(datadir)/nix/corepkgs/nar
$(INSTALL_PROGRAM) unnar.sh $(DESTDIR)$(datadir)/nix/corepkgs/nar
include ../../substitute.mk
EXTRA_DIST = nar.nix nar.sh.in
EXTRA_DIST = nar.nix nar.sh.in unnar.nix unnar.sh.in
+3 -2
View File
@@ -1,5 +1,6 @@
{system, path, hashAlgo}: derivation {
{system, path}: derivation {
name = "nar";
builder = ./nar.sh;
inherit system path hashAlgo;
system = system;
path = path;
}
+10 -9
View File
@@ -1,14 +1,15 @@
#! @shell@ -e
# !!! impure; fix this
export PATH=/bin:/usr/bin
echo "packing $path into $out..."
@coreutils@/mkdir $out
dst=$out/tmp.nar.bz2
@bindir@/nix-store --dump "$path" > tmp
mkdir $out
dst=$out/$(basename $path).nar.bz2
@bindir@/nix-store --dump "$path" | @bzip2@ > $dst
@bzip2@ < tmp > $dst
if test "${PIPESTATUS[0]}" != "0"; then exit 1; fi
@bindir@/nix-hash -vvvvv --flat --type $hashAlgo --base32 tmp > $out/nar-hash
@bindir@/nix-hash --flat --type $hashAlgo --base32 $dst > $out/narbz2-hash
@coreutils@/mv $out/tmp.nar.bz2 $out/$(@coreutils@/cat $out/narbz2-hash).nar.bz2
md5=$(md5sum -b $dst | cut -c1-32)
if test $? != 0; then exit 1; fi
echo $md5 > $out/md5
+7
View File
@@ -0,0 +1,7 @@
{system, narFile, outPath}: derivation {
name = "unnar";
builder = ./unnar.sh;
system = system;
narFile = narFile;
outPath = outPath;
}
+4
View File
@@ -0,0 +1,4 @@
#! @shell@ -e
echo "unpacking $narFile to $out..."
@bunzip2@ < $narFile | @bindir@/nix-store --restore "$out"
+11
View File
@@ -0,0 +1,11 @@
all-local: builder.sh
install-exec-local:
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
include ../../substitute.mk
EXTRA_DIST = default.nix builder.sh.in
+34
View File
@@ -0,0 +1,34 @@
#! @shell@ -e
export PATH=/bin:/usr/bin
mkdir $out
cat > $out/fetch <<EOF
#! @shell@ -e
export PATH=/bin:/usr/bin
echo "downloading \$2..."
export PRINT_PATH=1
result=(\$(@bindir@/nix-prefetch-url \$2))
hash=\${result[0]}
path=\${result[1]}
if test "\$hash" != "\$3"; then
echo "hash is \$hash, expected \$3"
exit 1
fi
echo "unpacking into \$1..."
if ! @bunzip2@ < "\$path" | @bindir@/nix-store --restore "\$1"; then
exit 1
fi
exit 0
EOF
chmod +x $out/fetch
+7
View File
@@ -0,0 +1,7 @@
{system}:
derivation {
name = "nix-pull";
builder = ./builder.sh;
inherit system;
}
+16 -20
View File
@@ -1,41 +1,37 @@
ENV = SGML_CATALOG_FILES=$(docbookcatalog)
ENV = SGML_CATALOG_FILES=$(docbookcatalog):$(docbookebnfcatalog)
XMLLINT = $(ENV) $(xmllint) $(xmlflags) --catalogs
XSLTPROC = $(ENV) $(xsltproc) $(xmlflags) --catalogs \
--param section.autolabel 1 \
--param section.label.includes.component.label 1 \
--param html.stylesheet \'style.css\' \
--param xref.with.number.and.title 1 \
--param toc.section.depth 3
--param xref.with.number.and.title 0
man1_MANS = nix-env.1 nix-build.1 nix-store.1 nix-instantiate.1 \
man1_MANS = nix-env.1 nix-store.1 nix-instantiate.1 \
nix-collect-garbage.1 nix-push.1 nix-pull.1 \
nix-prefetch-url.1 nix-channel.1
nix-prefetch-url.1
FIGURES = figures/user-environments.png
MANUAL_SRCS = manual.xml introduction.xml installation.xml \
SOURCES = manual.xml introduction.xml installation.xml \
package-management.xml writing-nix-expressions.xml \
build-farm.xml \
$(man1_MANS:.1=.xml) \
troubleshooting.xml bugs.xml opt-common.xml opt-common-syn.xml \
env-common.xml quick-start.xml nix-lang-ref.xml glossary.xml \
conf-file.xml \
style.css images
quick-start.xml nix-lang-ref.xml style.css images
manual.is-valid: $(MANUAL_SRCS) version.txt
$(XMLLINT) --xinclude $< | $(XMLLINT) --noout --nonet --valid -
manual.is-valid: $(SOURCES) version.xml
$(XMLLINT) --noout --valid manual.xml
touch $@
version.txt:
echo -n $(VERSION) > version.txt
version.xml:
echo -n $(VERSION) > version.xml
man $(MANS): $(MANUAL_SRCS) manual.is-valid
$(XSLTPROC) --nonet --xinclude $(docbookxsl)/manpages/docbook.xsl manual.xml
man $(MANS): $(SOURCES) manual.is-valid
$(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl manual.xml
manual.html: $(MANUAL_SRCS) manual.is-valid images
$(XSLTPROC) --nonet --xinclude --output manual.html \
$(docbookxsl)/html/docbook.xsl manual.xml
manual.html: $(SOURCES) manual.is-valid images
$(XSLTPROC) --output manual.html $(docbookxsl)/html/docbook.xsl manual.xml
all-local: manual.html
@@ -54,8 +50,8 @@ images:
cp $(docbookxsl)/images/callouts/*.png images/callouts
chmod +w -R images
KEEP = manual.html manual.is-valid version.txt $(MANS)
KEEP = manual.html manual.is-valid version.xml $(MANS)
EXTRA_DIST = $(MANUAL_SRCS) $(FIGURES) $(KEEP)
EXTRA_DIST = $(SOURCES) $(FIGURES) $(KEEP)
DISTCLEANFILES = $(KEEP)
+87 -25
View File
@@ -2,25 +2,90 @@
<itemizedlist>
<listitem><para>The man-pages generated from the DocBook documentation
are ugly.</para></listitem>
<listitem>
<para>
The man-pages generated from the DocBook documentation are ugly.
</para>
</listitem>
<listitem><para>Generations properly form a tree. E.g., if after
switching to generation 39, we perform an installation action, a
generation 43 is created which is a descendant of 39, not 42. So a
rollback from 43 ought to go back to 39. This is not currently
implemented; generations form a linear sequence.</para></listitem>
<listitem>
<para>
Generations properly form a tree. E.g., if after switching to
generation 39, we perform an installation action, a generation
43 is created which is a descendant of 39, not 42. So a
rollback from 43 ought to go back to 39. This is not
currently implemented; generations form a linear sequence.
</para>
</listitem>
<listitem><para><emphasis>Build management.</emphasis> In principle it
is already possible to do build management using Nix (by writing
builders that perform appropriate build steps), but the Nix expression
language is not yet powerful enough to make this pleasant (?). The
language should be extended with features from the <ulink
url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
Another interesting idea is to write a <command>make</command>
implementation that uses Nix as a back-end to support <ulink
url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink>
build files.</para></listitem>
<listitem>
<para>
Unify the concepts of successors and substitutes into a
general notion of <emphasis>equivalent expressions</emphasis>.
Expressions are equivalent if they have the same target paths
with the same identifiers. However, even though they are
functionally equivalent, they may differ stronly with respect
to their <emphasis>performance characteristics</emphasis>.
For example, realising a closure expression is more efficient
that realising the derivation expression from which it was
produced. On the other hand, distributing sources may be more
efficient (storage- or bandwidth-wise) than distributing
binaries. So we need to be able to attach weigths or
priorities or performance annotations to expressions; Nix can
then choose the most efficient expression dependent on the
context.
</para>
</listitem>
<listitem>
<para>
<emphasis>Build management.</emphasis> In principle it is already
possible to do build management using Nix (by writing builders that
perform appropriate build steps), but the Nix expression language is
not yet powerful enough to make this pleasant (?). The language should
be extended with features from the <ulink
url='http://www.cs.uu.nl/~eelco/maak/'>Maak build manager</ulink>.
Another interesting idea is to write a <command>make</command>
implementation that uses Nix as a back-end to support <ulink
url='http://www.research.att.com/~bs/bs_faq.html#legacy'>legacy</ulink>
build files.
</para>
</listitem>
<listitem>
<para>
There are race conditions between the garbage collector and
other Nix tools. For instance, when we run
<command>nix-env</command> to build and install a derivation
and run the garbage collector at the same time, the garbage
collector may kick in exactly between the build and
installation steps, i.e., before the newly built derivation
has become reachable from a root of the garbage collector.
</para>
<para>
One solution would be for these programs to properly register
temporary roots for the collector. Another would be to use
stop-the-world garbage collection: if any tool is running, the
garbage collector blocks, and vice versa. These solutions do
not solve the situation where multiple tools are involved,
e.g.,
<screen>
$ nix-store -r $(nix-instantiate foo.nix)</screen>
since even if <command>nix-instantiate</command> where to
register a temporary root, it would be released by the time
<command>nix-store</command> is started. A solution would be
to write the intermediate value to a file that is used as a
root to the collector, e.g.,
<screen>
$ nix-instantiate foo.nix > /nix/var/nix/roots/bla
$ nix-store -r $(cat /nix/var/nix/roots/bla)</screen>
</para>
</listitem>
<listitem><para>For security, <command>nix-push</command> manifests
should be digitally signed, and <command>nix-pull</command> should
@@ -29,18 +94,15 @@ need to be signed, since the manifest contains cryptographic hashes of
these files (and <filename>fetchurl.nix</filename> checks
them).</para></listitem>
<listitem><para>We should switch away from MD5, since it has been
more-or-less cracked. We don't currently depend very much on the
collision-resistance of MD5, but we will once we start sharing build
results between users.</para></listitem>
<listitem><para>It would be useful to have an option in
<command>nix-env --delete-generations</command> to remove non-current
generations older than a certain age.</para></listitem>
<listitem><para>There should be a flexible way to change the user
environment builder. Currently, you have to replace
<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>,
which is hard-coded into <command>nix-env</command>. Also, the
default builder should be more powerful. For instance, there should
be some way to specify priorities to resolve
collisions.</para></listitem>
</itemizedlist>
</appendix>
+1 -4
View File
@@ -65,10 +65,7 @@ will call whenever it wants to build a derivation. The build hook
will perform it in the usual way if possible, or it can accept it, in
which case it is responsible for somehow getting the inputs of the
build to another machine, doing the build there, and getting the
results back. The details of the build hook protocol are described in
the documentation of the <link
linkend="envar-build-hook"><envar>NIX_BUILD_HOOK</envar>
variable</link>.</para>
results back.</para>
<example id='ex-remote-systems'><title>Remote machine configuration:
<filename>remote-systems.conf</filename></title>
-82
View File
@@ -1,82 +0,0 @@
<sect1 id="sec-conf-file"><title>Nix configuration file</title>
<para>A number of persistent settings of Nix are stored in the file
<filename><replaceable>prefix</replaceable>/etc/nix/nix.conf</filename>.
This file is a list of <literal><replaceable>name</replaceable> =
<replaceable>value</replaceable></literal> pairs, one per line.
Comments start with a <literal>#</literal> character. An example
configuration file is shown in <xref linkend="ex-nix-conf" />.</para>
<example id='ex-nix-conf'><title>Nix configuration file</title>
<programlisting>
gc-keep-outputs = true # Nice for developers
gc-keep-derivations = true # Idem
env-keep-derivations = false
</programlisting>
</example>
<para>The following variables are currently available:
<variablelist>
<varlistentry id="conf-gc-keep-outputs"><term><literal>gc-keep-outputs</literal></term>
<listitem><para>If <literal>true</literal>, the garbage collector
will keep the outputs of non-garbage derivations. If
<literal>false</literal> (default), outputs will be deleted unless
they are GC roots themselves (or reachable from other roots).</para>
<para>In general, outputs must be registered as roots separately.
However, even if the output of a derivation is registered as a
root, the collector will still delete store paths that are used
only at build time (e.g., the C compiler, or source tarballs
downloaded from the network). To prevent it from doing so, set
this option to <literal>true</literal>.</para></listitem>
</varlistentry>
<varlistentry id="conf-gc-keep-derivations"><term><literal>gc-keep-derivations</literal></term>
<listitem><para>If <literal>true</literal> (default), the garbage
collector will keep the derivations from which non-garbage store
paths were built. If <literal>false</literal>, they will be
deleted unless explicitly registered as a root (or reachable from
other roots).</para>
<para>Keeping derivation around is useful for querying and
traceability (e.g., it allows you to ask with what dependencies or
options a store path was built), so by default this option is on.
Turn it off to safe a bit of disk space (or a lot if
<literal>gc-keep-outputs</literal> is also turned on).</para></listitem>
</varlistentry>
<varlistentry><term><literal>env-keep-derivations</literal></term>
<listitem><para>If <literal>false</literal> (default), derivations
are not stored in Nix user environments. That is, the derivation
any build-time-only dependencies may be garbage-collected.</para>
<para>If <literal>true</literal>, when you add a Nix derivation to
a user environment, the path of the derivation is stored in the
user environment. Thus, the derivation will not be
garbage-collected until the user environment generation is deleted
(<command>nix-env --delete-generations</command>). To prevent
build-time-only dependencies from being collected, you should also
turn on <literal>gc-keep-outputs</literal>.</para>
<para>The difference between this option and
<literal>gc-keep-derivations</literal> is that this one is
“sticky”: it applies to any user environment created while this
option was enabled, while <literal>gc-keep-derivations</literal>
only applies at the moment the garbage collector is
run.</para></listitem>
</varlistentry>
</variablelist>
</para>
</sect1>
-274
View File
@@ -1,274 +0,0 @@
<sect1 id="sec-common-env"><title>Common environment variables</title>
<para>Most Nix commands interpret the following environment variables:</para>
<variablelist>
<varlistentry><term><envar>NIX_ROOT</envar></term>
<listitem><para>If <envar>NIX_ROOT</envar> is set, the Nix command
will on startup perform a <function>chroot()</function> to the
specified directory. This is useful in certain bootstrapping
situations (e.g., when installing a Nix installation onto a hard
disk from CD-ROM).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term>
<listitem>
<para>Normally, the Nix store directory (typically
<filename>/nix/store</filename>) is not allowed to contain any
symlink components. This is to prevent “impure” builds. Builders
sometimes “canonicalise” paths by resolving all symlink components.
Thus, builds on different machines (with
<filename>/nix/store</filename> resolving to different locations)
could yield different results. This is generally not a problem,
except when builds are deployed to machines where
<filename>/nix/store</filename> resolves differently. If you are
sure that you’re not going to do that, you can set
<envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para>
<para>Note that if you’re symlinking the Nix store so that you can
put it on another file system than the root file system, on Linux
you’re better off using <literal>bind</literal> mount points, e.g.,
<screen>
$ mkdir /nix
$ mount -o bind /mnt/otherdisk/nix /nix</screen>
Consult the <citerefentry><refentrytitle>mount</refentrytitle>
<manvolnum>8</manvolnum></citerefentry> manual page for details.</para>
</listitem>
</varlistentry>
<varlistentry><term><envar>NIX_STORE_DIR</envar></term>
<listitem><para>Overrides the location of the Nix store (default
<filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_DATA_DIR</envar></term>
<listitem><para>Overrides the location of the Nix static data
directory (default
<filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_LOG_DIR</envar></term>
<listitem><para>Overrides the location of the Nix log directory
(default <filename><replaceable>prefix</replaceable>/log/nix</filename>).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_STATE_DIR</envar></term>
<listitem><para>Overrides the location of the Nix state directory
(default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_DB_DIR</envar></term>
<listitem><para>Overrides the location of the Nix database (default
<filename><replaceable>$NIX_STATE_DIR</replaceable>/db</filename>, i.e.,
<filename><replaceable>prefix</replaceable>/var/nix/db</filename>).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_CONF_DIR</envar></term>
<listitem><para>Overrides the location of the Nix configuration
directory (default
<filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem>
</varlistentry>
<varlistentry><term><envar>NIX_LOG_TYPE</envar></term>
<listitem><para>Equivalent to the <link
linkend="opt-log-type"><option>--log-type</option>
option</link>.</para></listitem>
</varlistentry>
<varlistentry><term><envar>TMPDIR</envar></term>
<listitem><para>Use the specified directory to store temporary
files. In particular, this includes temporary build directories;
these can take up substantial amounts of disk space. The default is
<filename>/tmp</filename>.</para></listitem>
</varlistentry>
<varlistentry id="envar-build-hook"><term><envar>NIX_BUILD_HOOK</envar></term>
<listitem>
<para>Specifies the location of the <emphasis>build hook</emphasis>,
which is a program (typically some script) that Nix will call
whenever it wants to build a derivation. This is used to implement
distributed builds (see <xref linkend="sec-distributed-builds"
/>). The protocol by which the calling Nix process and the build
hook communicate is as follows.</para>
<para>The build hook is called with the following command-line
arguments:
<orderedlist>
<listitem><para>A boolean value <literal>0</literal> or
<literal>1</literal> specifying whether Nix can locally execute
more builds, as per the <link
linkend="opt-max-jobs"><option>--max-jobs</option> option</link>.
The purpose of this argument is to allow the hook to not have to
maintain bookkeeping for the local machine.</para></listitem>
<listitem><para>The Nix platform identifier for the local machine
(e.g., <literal>i686-linux</literal>).</para></listitem>
<listitem><para>The Nix platform identifier for the derivation,
i.e., its <link linkend="attr-system"><varname>system</varname>
attribute</link>.</para></listitem>
<listitem><para>The store path of the derivation.</para></listitem>
</orderedlist>
</para>
<para>On the basis of this information, and whatever persistent
state the build hook keeps about other machines and their current
load, it has to decide what to do with the build. It should print
out on file descriptor 3 one of the following responses (terminated
by a newline, <literal>"\n"</literal>):
<variablelist>
<varlistentry><term><literal>decline</literal></term>
<listitem><para>The build hook is not willing or able to perform
the build; the calling Nix process should do the build itself,
if possible.</para></listitem>
</varlistentry>
<varlistentry><term><literal>postpone</literal></term>
<listitem><para>The build hook cannot perform the build now, but
can do so in the future (e.g., because all available build slots
on remote machines are in use). The calling Nix process should
postpone this build until at least one currently running build
has terminated.</para></listitem>
</varlistentry>
<varlistentry><term><literal>accept</literal></term>
<listitem><para>The build hook has accepted the
build.</para></listitem>
</varlistentry>
</variablelist>
</para>
<para>If the build hook accepts the build, it is possible that it is
no longer necessary to do the build because some other process has
performed the build in the meantime. To prevent races, the hook
must read from file descriptor 4 a single line that tells it whether
to continue:
<variablelist>
<varlistentry><term><literal>cancel</literal></term>
<listitem><para>The build has already been done, so the hook
should exit.</para></listitem>
</varlistentry>
<varlistentry><term><literal>okay</literal></term>
<listitem><para>The hook should proceed with the build. At this
point, the calling Nix process has acquired locks on the output
path, so no other Nix process will perform the
build.</para></listitem>
</varlistentry>
</variablelist>
</para>
<para>If the hook has been told to proceed, Nix will store in the
hook’s current directory a number of text files that contain
information about the derivation:
<variablelist>
<varlistentry><term><filename>inputs</filename></term>
<listitem><para>The set of store paths that are inputs to the
build process (one per line). These have to be copied
<emphasis>to</emphasis> the remote machine (in addition to the
store derivation itself).</para></listitem>
</varlistentry>
<varlistentry><term><filename>outputs</filename></term>
<listitem><para>The set of store paths that are outputs of the
derivation (one per line). These have to be copied
<emphasis>from</emphasis> the remote machine if the build
succeeds.</para></listitem>
</varlistentry>
<varlistentry><term><filename>references</filename></term>
<listitem><para>The reference graph of the inputs, in the format
accepted by the command <command>nix-store
--register-validity</command>. It is necessary to run this
command on the remote machine after copying the inputs to inform
Nix on the remote machine that the inputs are valid
paths.</para></listitem>
</varlistentry>
</variablelist>
</para>
<para>The hook should copy the inputs to the remote machine,
register the validity of the inputs, perform the remote build, and
copy the outputs back to the local machine. An exit code other than
<literal>0</literal> indicates that the hook has failed.</para>
</listitem>
</varlistentry>
</variablelist>
</sect1>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 83 KiB

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.
-163
View File
@@ -1,163 +0,0 @@
<appendix><title>Glossary</title>
<glosslist>
<glossentry id="gloss-derivation"><glossterm>derivation</glossterm>
<glossdef><para>A description of a build action. The result of a
derivation is a store object. Derivations are typically specified
in Nix expressions using the <link
linkend="ssec-derivation"><function>derivation</function>
primitive</link>. These are translated into low-level
<emphasis>store derivations</emphasis> (implicitly by
<command>nix-env</command> and <command>nix-build</command>, or
explicitly by <command>nix-instantiate</command>).</para></glossdef>
</glossentry>
<glossentry><glossterm>store</glossterm>
<glossdef><para>The location in the file system where store objects
live. Typically <filename>/nix/store</filename>.</para></glossdef>
</glossentry>
<glossentry><glossterm>store path</glossterm>
<glossdef><para>The location in the file system of a store object,
i.e., an immediate child of the Nix store
directory.</para></glossdef>
</glossentry>
<glossentry><glossterm>store object</glossterm>
<glossdef><para>A file that is an immediate child of the Nix store
directory. These can be regular files, but also entire directory
trees. Store objects can be sources (objects copied from outside of
the store), derivation outputs (objects produced by running a build
action), or derivations (files describing a build
action).</para></glossdef>
</glossentry>
<glossentry id="gloss-substitute"><glossterm>substitute</glossterm>
<glossdef><para>A substitute is a command invocation stored in the
Nix database that describes how to build a store object, bypassing
normal the build mechanism (i.e., derivations). Typically, the
substitute builds the store object by downloading a pre-built
version of the store object from some server.</para></glossdef>
</glossentry>
<glossentry><glossterm>purity</glossterm>
<glossdef><para>The assumption that equal Nix derivations when run
always produce the same output. This cannot be guaranteed in
general (e.g., a builder can rely on external inputs such as the
network or the system time) but the Nix model assumes
it.</para></glossdef>
</glossentry>
<glossentry><glossterm>Nix expression</glossterm>
<glossdef><para>A high-level description of software components and
compositions thereof. Deploying software using Nix entails writing
Nix expressions for your components. Nix expressions are translated
to derivations that are stored in the Nix store. These derivations
can then be built.</para></glossdef>
</glossentry>
<glossentry id="gloss-reference"><glossterm>reference</glossterm>
<glossdef><para>A store path <varname>P</varname> is said to have a
reference to a store path <varname>Q</varname> if the store object
at <varname>P</varname> contains the path <varname>Q</varname>
somewhere. This implies than an execution involving
<varname>P</varname> potentially needs <varname>Q</varname> to be
present. The <emphasis>references</emphasis> of a store path are
the set of store paths to which it has a reference.</para></glossdef>
</glossentry>
<glossentry id="gloss-closure"><glossterm>closure</glossterm>
<glossdef><para>The closure of a store path is the set of store
paths that are directly or indirectly “reachable” from that store
path; that is, it’s the closure of the path under the <link
linkend="gloss-reference">references</link> relation. For instance,
if the store object at path <varname>P</varname> contains a
reference to path <varname>Q</varname>, then <varname>Q</varname> is
in the closure of <varname>P</varname>. For correct deployment it
is necessary to deploy whole closures, since otherwise at runtime
files could be missing. The command <command>nix-store
-qR</command> prints out closures of store paths.</para></glossdef>
</glossentry>
<glossentry id="gloss-output-path"><glossterm>output path</glossterm>
<glossdef><para>A store path produced by a derivation.</para></glossdef>
</glossentry>
<glossentry id="gloss-deriver"><glossterm>deriver</glossterm>
<glossdef><para>The deriver of an <link
linkend="gloss-output-path">output path</link> is the store
derivation that built it.</para></glossdef>
</glossentry>
<glossentry id="gloss-validity"><glossterm>validity</glossterm>
<glossdef><para>A store path is considered
<emphasis>valid</emphasis> if it exists in the file system, is
listed in the Nix database as being valid, and if all paths in its
closure are also valid.</para></glossdef>
</glossentry>
<glossentry id="gloss-user-env"><glossterm>user environment</glossterm>
<glossdef><para>An automatically generated store object that
consists of a set of symlinks to “active” applications, i.e., other
store paths. These are generated automatically by <link
linkend="sec-nix-env"><command>nix-env</command></link>. See <xref
linkend="sec-profiles" />.</para>
</glossdef>
</glossentry>
<glossentry id="gloss-profile"><glossterm>profile</glossterm>
<glossdef><para>A symlink to the current <link
linkend="gloss-user-env">user environment</link> of a user, e.g.,
<filename>/nix/var/nix/profiles/default</filename>.</para></glossdef>
</glossentry>
</glosslist>
</appendix>
+2 -2
View File
@@ -5,8 +5,8 @@
<para>The easiest way to obtain Nix is to download a <ulink
url='http://www.cs.uu.nl/groups/ST/Trace/Nix'>source
distribution</ulink>. RPMs for Red Hat, SuSE, and Fedore Core are
also available.</para>
distribution</ulink>. RPMs for Red Hat 9 are also available. These
distributions are generated automatically.</para>
<para>Alternatively, the most recent sources of Nix can be obtained
from its <ulink
+7 -14
View File
@@ -75,7 +75,7 @@ be adapted easily to achieve similar policies.</para></listitem>
<listitem><para>Nix component builds aim to be <quote>pure</quote>;
that is, unaffected by anything other than the declared dependencies.
This means that if a component was built successfully once, it can be
This means that if a component was built succesfully once, it can be
rebuilt again on another machine and the result will be the same. We
cannot <emphasis>guarantee</emphasis> this (e.g., if the build depends
on the time-of-day), but Nix (and the tools in the Nix Packages
@@ -113,13 +113,6 @@ software deployment also apply here: for instance, the ability
trivially to have multiple configurations at the same time, or the
ability to do rollbacks.</para></listitem>
<listitem><para>Nix can efficiently upgrade between different versions
of a component through <emphasis>binary patching</emphasis>. If
patches are available on a server, and you try to install a new
version of some component, Nix will automatically apply a patch (or
sequence of patches), if available, to transform the installed
component into the new version.</para></listitem>
</itemizedlist>
</para>
@@ -130,7 +123,11 @@ collection. It also discusses some advanced topics, such as setting
up a Nix-based build farm, and doing service deployment using
Nix.</para>
<note><para>Some background information on Nix can be found in three
<warning><para>This manual is a work in progress. It's quite likely
to be incomplete, inconsistent with the current implementation, or
simply wrong.</para></warning>
<note><para>Some background information on Nix can be found in two
papers. The ICSE 2004 paper <ulink
url='http://www.cs.uu.nl/~eelco/pubs/immdsd-icse2004-final.pdf'><citetitle>Imposing
a Memory Management Discipline on Software
@@ -141,10 +138,6 @@ different versions and variants of packages. The LISA 2004 paper
url='http://www.cs.uu.nl/~eelco/pubs/nspfssd-lisa2004-final.pdf'><citetitle>Nix:
A Safe and Policy-Free System for Software
Deployment</citetitle></ulink> gives a more general discussion of Nix
from a system-administration perspective. The CBSE 2005 paper <ulink
url='http://www.cs.uu.nl/~eelco/pubs/eupfcdm-cbse2005-final.pdf'><citetitle>Efficient
Upgrading in a Purely Functional Component Deployment Model
</citetitle></ulink> is about transparent patch deployment in
Nix.</para></note>
from a system-administration perspective.</para></note>
</chapter>
+39 -35
View File
@@ -3,14 +3,31 @@
PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
"http://www.docbook.org/xml/4.3/docbook-xml-4.3.zip"
[
<!ENTITY introduction SYSTEM "introduction.xml">
<!ENTITY quick-start SYSTEM "quick-start.xml">
<!ENTITY installation SYSTEM "installation.xml">
<!ENTITY package-management SYSTEM "package-management.xml">
<!ENTITY writing-nix-expressions SYSTEM "writing-nix-expressions.xml">
<!ENTITY build-farm SYSTEM "build-farm.xml">
<!ENTITY opt-common SYSTEM "opt-common.xml">
<!ENTITY opt-common-syn SYSTEM "opt-common-syn.xml">
<!ENTITY nix-env SYSTEM "nix-env.xml">
<!ENTITY nix-store SYSTEM "nix-store.xml">
<!ENTITY nix-instantiate SYSTEM "nix-instantiate.xml">
<!ENTITY nix-collect-garbage SYSTEM "nix-collect-garbage.xml">
<!ENTITY nix-push SYSTEM "nix-push.xml">
<!ENTITY nix-pull SYSTEM "nix-pull.xml">
<!ENTITY nix-prefetch-url SYSTEM "nix-prefetch-url.xml">
<!-- <!ENTITY nix-lang-ref SYSTEM "nix-lang-ref.xml"> -->
<!ENTITY troubleshooting SYSTEM "troubleshooting.xml">
<!ENTITY bugs SYSTEM "bugs.xml">
<!ENTITY version SYSTEM "version.xml">
]>
<book>
<title>Nix User's Guide</title>
<subtitle>Draft (Version <xi:include
xmlns:xi="http://www.w3.org/2001/XInclude"
href="version.txt" parse="text" />)</subtitle>
<subtitle>Draft (Version &version;)</subtitle>
<bookinfo>
<author>
@@ -19,65 +36,52 @@
</author>
<copyright>
<year>2004</year>
<year>2005</year>
<holder>Eelco Dolstra</holder>
</copyright>
</bookinfo>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="introduction.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="quick-start.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="installation.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="package-management.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="writing-nix-expressions.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="build-farm.xml" />
&introduction;
&quick-start;
&installation;
&package-management;
&writing-nix-expressions;
&build-farm;
<appendix>
<title>Command Reference</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="env-common.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="conf-file.xml" />
<sect1 id="sec-nix-env">
<sect1>
<title>nix-env</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-env.xml" />
</sect1>
<sect1 id="sec-nix-build">
<title>nix-build</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-build.xml" />
&nix-env;
</sect1>
<sect1>
<title>nix-store</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-store.xml" />
&nix-store;
</sect1>
<sect1 id="sec-nix-instantiate">
<sect1>
<title>nix-instantiate</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-instantiate.xml" />
&nix-instantiate;
</sect1>
<sect1>
<title>nix-collect-garbage</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-collect-garbage.xml" />
</sect1>
<sect1 id="sec-nix-channel">
<title>nix-channel</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-channel.xml" />
&nix-collect-garbage;
</sect1>
<sect1>
<title>nix-push</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-push.xml" />
&nix-push;
</sect1>
<sect1>
<title>nix-pull</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-pull.xml" />
&nix-pull;
</sect1>
<sect1>
<title>nix-prefetch-url</title>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="nix-prefetch-url.xml" />
&nix-prefetch-url;
</sect1>
</appendix>
<!-- &nix-lang-ref; -->
<!-- &nix-lang-ref; -->
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="troubleshooting.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="bugs.xml" />
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="glossary.xml" />
&troubleshooting;
&bugs;
</book>
-74
View File
@@ -1,74 +0,0 @@
<refentry>
<refnamediv>
<refname>nix-build</refname>
<refpurpose>build a Nix expression</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-build</command>
<arg><option>--add-drv-link</option></arg>
<arg><option>--no-link</option></arg>
<arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsection><title>Description</title>
<para>The <command>nix-build</command> command builds the derivations
described by the Nix expressions in <replaceable>paths</replaceable>.
If the build succeeds, it places a symlink to the result in the
current directory. The symlink is called <filename>result</filename>.
If there are multiple Nix expressions, or the Nix expressions evaluate
to multiple derivations, multiple sequentially numbered symlinks are
created (<filename>result</filename>, <filename>result-2</filename>,
and so on).</para>
<note><para><command>nix-build</command> is essentially a wrapper
around <link
linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>
(to translate a high-level Nix expression to a low-level store
derivation) and <link
linkend="rsec-nix-store-realise"><command>nix-store
--realise</command></link> (to build the store
derivation).</para></note>
<warning><para>The result of the build is automatically registered as
a root of the Nix garbage collector. This root disappears
automatically when the <filename>result</filename> symlink is deleted
or renamed. So don’t rename the symlink.</para></warning>
</refsection>
<refsection><title>Options</title>
<variablelist>
<varlistentry><term><option>--add-drv-link</option></term>
<listitem><para>Add a symlink in the current directory to the
store derivation produced by <command>nix-instantiate</command>.
The symlink is called <filename>derivation</filename> (which is
numbered in the case of multiple derivations). The derivation is
a root of the garbage collector until the symlink is deleted or
renamed.</para></listitem>
</varlistentry>
<varlistentry><term><option>--no-link</option></term>
<listitem><para>Do not create a symlink to the output path. Note
that as a result the output does not become a root of the garbage
collector, and so might be deleted by <command>nix-store
--gc</command>.</para></listitem>
</varlistentry>
</variablelist>
</refsection>
</refentry>
-83
View File
@@ -1,83 +0,0 @@
<refentry>
<refnamediv>
<refname>nix-channel</refname>
<refpurpose>manage Nix channels</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-channel</command>
<group choice='req'>
<arg choice='plain'><option>--add</option> <replaceable>url</replaceable></arg>
<arg choice='plain'><option>--remove</option> <replaceable>url</replaceable></arg>
<arg choice='plain'><option>--list</option></arg>
<arg choice='plain'><option>--update</option></arg>
</group>
</cmdsynopsis>
</refsynopsisdiv>
<refsection><title>Description</title>
<para>A Nix channel is mechanism that allows you to automatically stay
up-to-date with a set of pre-built Nix expressions. A Nix channel is
just a URL that points to a place that contains a set of Nix
expressions, as well as a <command>nix-push</command> manifest. See
also <xref linkend="sec-channels" />.</para>
<para>This command has the following operations:
<variablelist>
<varlistentry><term><option>--add</option> <replaceable>url</replaceable></term>
<listitem><para>Adds <replaceable>url</replaceable> to the list of
subscribed channels.</para></listitem>
</varlistentry>
<varlistentry><term><option>--remove</option> <replaceable>url</replaceable></term>
<listitem><para>Removes <replaceable>url</replaceable> from the
list of subscribed channels.</para></listitem>
</varlistentry>
<varlistentry><term><option>--list</option></term>
<listitem><para>Prints the URLs of all subscribed channels on
standard output.</para></listitem>
</varlistentry>
<varlistentry><term><option>--update</option></term>
<listitem><para>Downloads the Nix expressions of all subscribed
channels, makes the conjunction of these the default for
<command>nix-env</command> operations (by calling <command>nix-env
-I</command>), and performs a <command>nix-pull</command> on the
manifests of all channels to make pre-built binaries
available.</para></listitem>
</varlistentry>
</variablelist>
</para>
<para>Note that <option>--add</option> and <option>--remove</option>
do not automatically perform an update.</para>
<para>The list of subscribed channels is stored in
<filename>~/.nix-channels</filename>.</para>
<para>A channel consists of two elements: a bzipped Tar archive
containing the Nix expressions, and a manifest created by
<command>nix-push</command>. These must be stored under
<literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal> and
<literal><replaceable>url</replaceable>/MANIFEST</literal>,
respectively.</para>
</refsection>
</refentry>
+80 -22
View File
@@ -1,29 +1,87 @@
<refentry>
<refnamediv>
<refname>nix-collect-garbage</refname>
<refpurpose>delete unreachable store paths</refpurpose>
</refnamediv>
<refnamediv>
<refname>nix-collect-garbage</refname>
<refpurpose>remove unreachable store paths</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-collect-garbage</command>
<group choice='opt'>
<arg choice='plain'><option>--print-roots</option></arg>
<arg choice='plain'><option>--print-live</option></arg>
<arg choice='plain'><option>--print-dead</option></arg>
<arg choice='plain'><option>--delete</option></arg>
</group>
</cmdsynopsis>
</refsynopsisdiv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-collect-garbage</command>
<group choice='opt'>
<arg choice='plain'><option>--print-live</option></arg>
<arg choice='plain'><option>--print-dead</option></arg>
</group>
<arg><option>--min-age</option> <replaceable>age</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsection><title>Description</title>
<refsection>
<title>Description</title>
<para>The command <command>nix-collect-garbage</command> is an
obsolete wrapper around <link
linkend="rsec-nix-store-gc"><command>nix-store
--gc</command></link>.</para>
<para>
The command <command>nix-collect-garbage</command> performs a
garbage collection on the Nix store: any paths in the Nix store
that are garbage (not reachable from a set of root store
expressions) are deleted.
</para>
</refsection>
<para>
The roots of the garbage collector are the store expressions
mentioned in the files in the directory
<filename><replaceable>prefix</replaceable>/var/nix/gcroots</filename>.
By default, the roots are all user environments in
<filename><replaceable>prefix</replaceable>/var/nix/profiles</filename>.
You can register other store expressions as roots by writing the
full path of the store expression to an arbitrary file in the
<filename>gcroots</filename> directory (or a subdirectory
thereof).
</para>
</refsection>
<refsection>
<title>Options</title>
<variablelist>
<varlistentry>
<term><option>--print-live</option> / <option>--print-dead</option></term>
<listitem>
<para>
These options cause the set of live or dead paths to be
printed, respectively, rather than performing an actual
garbage collector. They correspond exactly with the
sub-operations in <command>nix-store
<option>--gc</option></command>.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--min-age</option> <replaceable>age</replaceable></term>
<listitem>
<para>
This option corresponds to the <option>--min-age</option>
option in <command>nix-store <option>--gc</option></command>.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsection>
<refsection>
<title>Examples</title>
<para>
To delete all unreachable paths, just do:
<screen>
$ nix-collect-garbage</screen>
</para>
</refsection>
</refentry>
+686 -677
View File
File diff suppressed because it is too large. Load diff
+69 -76
View File
@@ -1,97 +1,90 @@
<refentry>
<refnamediv>
<refname>nix-instantiate</refname>
<refpurpose>instantiate store derivations from Nix expressions</refpurpose>
</refnamediv>
<refnamediv>
<refname>nix-instantiate</refname>
<refpurpose>instantiate store expressions from Nix expressions</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-instantiate</command>
<xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xpointer(/nop/*)" />
<arg><option>--add-root</option> <replaceable>path</replaceable></arg>
<arg><option>--indirect</option></arg>
<group choice='opt'>
<arg choice='plain'><option>--parse-only</option></arg>
<arg choice='plain'><option>--eval-only</option></arg>
</group>
<arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-instantiate</command>
&opt-common-syn;
<group choice='opt'>
<arg choice='plain'><option>--parse-only</option></arg>
<arg choice='plain'><option>--eval-only</option></arg>
</group>
<arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsection>
<title>Description</title>
<refsection><title>Description</title>
<para>
The command <command>nix-instantiate</command> generates
(low-level) store expressions from (high-level) Nix expressions.
It loads and evaluates the Nix expressions in each of
<replaceable>files</replaceable>. Each top-level expression
should evaluate to a derivation, a list of derivations, or a set
of derivations. The paths of the resulting store expressions
are printed on standard output.
</para>
<para>The command <command>nix-instantiate</command> generates <link
linkend="gloss-derivation">store derivations</link> from (high-level)
Nix expressions. It loads and evaluates the Nix expressions in each
of <replaceable>files</replaceable>. Each top-level expression should
evaluate to a derivation, a list of derivations, or a set of
derivations. The paths of the resulting store derivations are printed
on standard output.</para>
<para>
This command is generally used for testing Nix expression before
they are used with <command>nix-env</command>.
</para>
<para>Most users and developers don’t need to use this command
(<command>nix-env</command> and <command>nix-build</command> perform
store derivation instantiation from Nix expressions automatically).
It is most commonly used for implementing new deployment
policies.</para>
</refsection>
<para>See also <xref linkend="sec-common-options" /> for a list of
common options.</para>
<refsection>
<title>Options</title>
</refsection>
<variablelist>
&opt-common;
<refsection><title>Options</title>
<variablelist>
<varlistentry>
<term><option>--add-root</option> <replaceable>path</replaceable></term>
<term><option>--indirect</option></term>
<listitem><para>See the <link linkend="opt-add-root">corresponding
options</link> in <command>nix-store</command>.</para></listitem>
</varlistentry>
<varlistentry><term><option>--parse-only</option></term>
<listitem><para>Just parse the input files, and print their
abstract syntax trees on standard output in ATerm
format.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--parse-only</option></term>
<listitem>
<para>
Just parse the input files, and print their abstract
syntax trees on standard output in ATerm format.
</para>
</listitem>
</varlistentry>
<varlistentry><term><option>--eval-only</option></term>
<listitem><para>Just parse and evaluate the input files, and print
the resulting values on standard output. No instantiation of
store derivations takes place.</para></listitem>
</varlistentry>
<varlistentry>
<term><option>--eval-only</option></term>
<listitem>
<para>
Just parse and evaluate the input files, and print the
resulting values on standard output. No instantiation of
store expressions takes place.
</para>
</listitem>
</varlistentry>
</variablelist>
</variablelist>
</refsection>
</refsection>
<refsection>
<title>Examples</title>
<refsection><title>Examples</title>
<screen>
$ nix-instantiate gcc.nix <lineannotation>(instantiate)</lineannotation>
/nix/store/468abdcb93aa22bb721142615b97698b-d-gcc-3.3.2.store
<screen>
$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation>
/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv
$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(build)</lineannotation>
$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation>
<replaceable>...</replaceable>
/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation>
$ nix-store -r $(nix-instantiate gcc.nix) <lineannotation>(print output path)</lineannotation>
/nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26
dr-xr-xr-x 2 eelco users 4096 1970-01-01 01:00 lib
$ ls -l /nix/store/9afa718cddfdfe94b5b9303d0430ceb1-gcc-3.3.2
dr-xr-xr-x 2 eelco users 360 2003-12-01 16:12 bin
dr-xr-xr-x 3 eelco users 72 2003-12-01 16:12 include
...</screen>
</refsection>
</refsection>
</refentry>
+42 -57
View File
@@ -1,69 +1,54 @@
<refentry>
<refnamediv>
<refname>nix-prefetch-url</refname>
<refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
</refnamediv>
<refnamediv>
<refname>nix-prefetch-url</refname>
<refpurpose>copy a file from a URL into the store and print its MD5 hash</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-prefetch-url</command>
<arg choice='plain'><replaceable>url</replaceable></arg>
<arg><replaceable>hash</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-prefetch-url</command>
<arg choice='plain'><replaceable>url</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsection>
<title>Description</title>
<refsection><title>Description</title>
<para>
The command <command>nix-prefetch-url</command> downloads the
file referenced by the URL <replaceable>url</replaceable>,
prints its MD5 cryptographic hash code, and copies it into the
Nix store. The file name in the store is
<filename><replaceable>hash</replaceable>-<replaceable>basename</replaceable></filename>,
where <replaceable>basename</replaceable> is everything
following the final slash in <replaceable>url</replaceable>.
</para>
<para>The command <command>nix-prefetch-url</command> downloads the
file referenced by the URL <replaceable>url</replaceable>, prints its
cryptographic hash, and copies it into the Nix store. The file name
in the store is
<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>,
where <replaceable>baseName</replaceable> is everything following the
final slash in <replaceable>url</replaceable>.</para>
<para>
This command is just a convenience to Nix expression writers.
Often a Nix expressions fetch some source distribution from the
network using the <literal>fetchurl</literal> expression
contained in <literal>nixpkgs</literal>. However,
<literal>fetchurl</literal> requires an MD5 hash. If you don't
know the hash, you would have to download the file first, and
then <literal>fetchurl</literal> would download it again when
you build your Nix expression. Since
<literal>fetchurl</literal> uses the same name for the
downloaded file as <command>nix-prefetch-url</command>, the
redundant download can be avoided.
</para>
<para>This command is just a convenience for Nix expression writers.
Often a Nix expression fetches some source distribution from the
network using the <literal>fetchurl</literal> expression contained in
Nixpkgs. However, <literal>fetchurl</literal> requires a
cryptographic hash. If you don't know the hash, you would have to
download the file first, and then <literal>fetchurl</literal> would
download it again when you build your Nix expression. Since
<literal>fetchurl</literal> uses the same name for the downloaded file
as <command>nix-prefetch-url</command>, the redundant download can be
avoided.</para>
</refsection>
<para>The environment variable <envar>NIX_HASH_ALGO</envar> specifies
which hash algorithm to use. It can be either <literal>md5</literal>,
<literal>sha1</literal>, or <literal>sha256</literal>. The default is
<literal>md5</literal>.</para>
<refsection>
<title>Examples</title>
<para>If <replaceable>hash</replaceable> is specified, then a download
is not performed if the Nix store already contains a file with the
same hash and base name. Otherwise, the file is downloaded, and an
error if signaled if the actual hash of the file does not match the
specified hash.</para>
<para>This command prints the hash on standard output. Additionally,
if the environment variable <envar>PRINT_PATH</envar> is set, the path
of the downloaded file in the Nix store is also printed.</para>
</refsection>
<refsection><title>Examples</title>
<screen>
<screen>
$ nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
0bbd1df101bc0294d440471e50feca71
$ PRINT_PATH=1 nix-prefetch-url ftp://ftp.nluug.nl/pub/gnu/make/make-3.80.tar.bz2
0bbd1df101bc0294d440471e50feca71
/nix/store/wvyz8ifdn7wyz1p3pqyn0ra45ka2l492-make-3.80.tar.bz2</screen>
</refsection>
...
file has hash 0bbd1df101bc0294d440471e50feca71
...</screen>
</refsection>
</refentry>
+108 -86
View File
@@ -1,116 +1,138 @@
<refentry>
<refnamediv>
<refname>nix-push</refname>
<refpurpose>push store paths onto a network cache</refpurpose>
</refnamediv>
<refnamediv>
<refname>nix-push</refname>
<refpurpose>push store paths onto a network cache</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-push</command>
<arg choice='plain'><replaceable>archives-put-url</replaceable></arg>
<arg choice='plain'><replaceable>archives-get-url</replaceable></arg>
<arg choice='plain'><replaceable>manifest-put-url</replaceable></arg>
<arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsynopsisdiv>
<cmdsynopsis>
<command>nix-push</command>
<group choice='req'>
<arg choice='req'>
<arg choice='plain'><replaceable>archivesPutURL</replaceable></arg>
<arg choice='plain'><replaceable>archivesGetURL</replaceable></arg>
<arg choice='plain'><replaceable>manifestPutURL</replaceable></arg>
</arg>
<arg choice='req'>
<arg choice='plain'><option>--copy</option></arg>
<arg choice='plain'><replaceable>archivesDir</replaceable></arg>
<arg choice='plain'><replaceable>manifestFile</replaceable></arg>
</arg>
</group>
<arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsection>
<title>Description</title>
<para>
The command <command>nix-push</command> builds a set of store
expressions (if necessary), and then packages and uploads all
store paths in the resulting closures to a server. A network
cache thus populated can subsequently be used to speed up
software deployment on other machines using the
<command>nix-pull</command> command.
</para>
<refsection><title>Description</title>
<para>The command <command>nix-push</command> builds a set of store
paths (if necessary), and then packages and uploads all store paths in
the resulting closures to a server. A network cache thus populated
can subsequently be used to speed up software deployment on other
machines using the <command>nix-pull</command> command.</para>
<para><command>nix-push</command> performs the following actions.
<para>
<command>nix-push</command> performs the following actions.
<orderedlist>
<orderedlist>
<listitem><para>Each path in <replaceable>paths</replaceable> is
realised (using <link
linkend='rsec-nix-store-realise'><literal>nix-store
--realise</literal></link>).</para></listitem>
<listitem>
<para>
The store expressions stored in
<replaceable>paths</replaceable> are realised (using
<literal>nix-store --realise</literal>).
</para>
</listitem>
<listitem><para>All paths in the closure of the store expressions
stored in <replaceable>paths</replaceable> are determined (using
<literal>nix-store --query --requisites
--include-outputs</literal>). It should be noted that since the
<option>--include-outputs</option> flag is used, you get a combined
source/binary distribution.</para></listitem>
<listitem>
<para>
All paths in the closure of the store expressions stored
in <replaceable>paths</replaceable> are determined (using
<literal>nix-store --query --requisites
--include-successors</literal>). It should be noted that
since the <option>--include-successors</option> flag is
used, if you specify a derivation store expression, you
get a combined source/binary distribution. If you only
want a binary distribution, you should specify the closure
store expression that result from realising these (see
below).
</para>
</listitem>
<listitem><para>All store paths determined in the previous step are
packaged and compressed into a <command>bzip</command>ped NAR
archive (extension <filename>.nar.bz2</filename>).</para></listitem>
<listitem>
<para>
All store paths determined in the previous step are
packaged and compressed into a <command>bzip</command>ped
NAR archive (extension <filename>.nar.bz2</filename>).
</para>
</listitem>
<listitem><para>A <emphasis>manifest</emphasis> is created that
contains information on the store paths, their eventual URLs in the
cache, and cryptographic hashes of the contents of the NAR
archives.</para></listitem>
<listitem>
<para>
A <emphasis>manifest</emphasis> is created that contains
information on the store paths, their eventual URLs in the
cache, and cryptographic hashes of the contents of the NAR
archives.
</para>
</listitem>
<listitem><para>Each store path is uploaded to the remote directory
specified by <replaceable>archivesPutURL</replaceable>. HTTP PUT
requests are used to do this. However, before a file
<varname>x</varname> is uploaded to
<literal><replaceable>archivesPutURL</replaceable>/<varname>x</varname></literal>,
<command>nix-push</command> first determines whether this upload is
unnecessary by issuing a HTTP HEAD request on
<literal><replaceable>archivesGetURL</replaceable>/<varname>x</varname></literal>.
This allows a cache to be shared between many partially overlapping
<command>nix-push</command> invocations. (We use two URLs because
the upload URL typically refers to a CGI script, while the download
URL just refers to a file system directory on the server.)</para></listitem>
<listitem>
<para>
Each store path is uploaded to the remote directory
specified by <replaceable>archives-put-url</replaceable>.
HTTP PUT requests are used to do this. However, before a
file <varname>x</varname> is uploaded to
<literal><replaceable>archives-put-url</replaceable>/<varname>x</varname></literal>,
<command>nix-push</command> first determines whether this
upload is unnecessary by issuing a HTTP HEAD request on
<literal><replaceable>archives-get-url</replaceable>/<varname>x</varname></literal>.
This allows a cache to be shared between many partially
overlapping <command>nix-push</command> invocations.
(We use two URLs because the upload URL typically
refers to a CGI script, while the download URL just refers
to a file system directory on the server.)
</para>
</listitem>
<listitem><para>The manifest is uploaded using an HTTP PUT request
to <replaceable>manifestPutURL</replaceable>. The corresponding
URL to download the manifest can then be used by
<command>nix-pull</command>.</para></listitem>
<listitem>
<para>
The manifest is uploaded using an HTTP PUT request to
<replaceable>manifest-put-url</replaceable>. The
corresponding URL to download the manifest can then be
used by <command>nix-pull</command>.
</para>
</listitem>
</orderedlist>
</para>
<para>TODO: <option>--copy</option></para>
</orderedlist>
</para>
</refsection>
</refsection>
<refsection>
<title>Examples</title>
<refsection><title>Examples</title>
<para>To upload files there typically is some CGI script on the server
side. This script should be be protected with a password. The
following example uploads the store paths resulting from building the
Nix expressions in <filename>foo.nix</filename>, passing appropriate
authentication information:
<para>
To upload files there typically is some CGI script on the server
side. This script should be be protected with a password. The
following example uploads the store paths resulting from
building the Nix expressions in <filename>foo.nix</filename>,
passing appropriate authentication information:
<screen>
<screen>
$ nix-push \
http://foo@bar:server.domain/cgi-bin/upload.pl/cache \
http://server.domain/cache \
http://foo@bar:server.domain/cgi-bin/upload.pl/MANIFEST \
$(nix-instantiate foo.nix)</screen>
This will push both sources and binaries (and any build-time
dependencies used in the build, such as compilers).</para>
This will push both sources and binaries (and any build-time
dependencies used in the build, such as compilers).
</para>
<para>If we just want to push binaries, not sources and build-time
dependencies, we can do:
<para>
If we just want to push binaries, not sources and build-time
dependencies, we can do:
<screen>
<screen>
$ nix-push <replaceable>urls</replaceable> $(nix-instantiate $(nix-store -r foo.nix))</screen>
</para>
</para>
</refsection>
</refsection>
</refentry>
+493 -606
View File
File diff suppressed because it is too large. Load diff
-5
View File
@@ -1,5 +1,3 @@
<nop>
<arg><option>--help</option></arg>
<arg><option>--version</option></arg>
<arg rep='repeat'><option>--verbose</option></arg>
@@ -19,6 +17,3 @@
<arg><option>-K</option></arg>
<arg><option>--fallback</option></arg>
<arg><option>--readonly-mode</option></arg>
<arg><option>--log-type</option> <replaceable>type</replaceable></arg>
</nop>
+163 -195
View File
@@ -1,216 +1,184 @@
<sect1 id="sec-common-options"><title>Common options</title>
<para>Most Nix commands accept the following command-line options:</para>
<variablelist>
<varlistentry><term><option>--help</option></term>
<listitem><para>Prints out a summary of the command syntax and
exits.</para></listitem>
</varlistentry>
<varlistentry><term><option>--version</option></term>
<listitem><para>Prints out the Nix version number on standard output
and exits.</para></listitem>
</varlistentry>
<varlistentry><term><option>--verbose</option></term>
<term><option>-v</option></term>
<varlistentry>
<term><option>--help</option></term>
<listitem>
<para>Increases the level of verbosity of diagnostic messages
printed on standard error. For each Nix operation, the information
printed on standard output is well-defined; any diagnostic
information is printed on standard error, never on standard
output.</para>
<para>This option may be specified repeatedly. Currently, the
following verbosity levels exist:</para>
<variablelist>
<varlistentry><term>0</term>
<listitem><para>“Errors only”: only print messages
explaining why the Nix invocation failed.</para></listitem>
</varlistentry>
<varlistentry><term>1</term>
<listitem><para>“Informational”: print
<emphasis>useful</emphasis> messages about what Nix is doing.
This is the default.</para></listitem>
</varlistentry>
<varlistentry><term>2</term>
<listitem><para>“Talkative”: print more informational
messages.</para></listitem>
</varlistentry>
<varlistentry><term>3</term>
<listitem><para>“Chatty”: print even more
informational messages.</para></listitem>
</varlistentry>
<varlistentry><term>4</term>
<listitem><para>“Debug”: print debug
information.</para></listitem>
</varlistentry>
<varlistentry><term>5</term>
<listitem><para>“Vomit”: print vast amounts of debug
information.</para></listitem>
</varlistentry>
</variablelist>
</listitem>
</varlistentry>
<varlistentry><term><option>--no-build-output</option></term>
<term><option>-Q</option></term>
<listitem><para>By default, output written by builders to standard
output and standard error is echoed to the Nix command's standard
error. This option suppresses this behaviour. Note that the
builder's standard output and error are always written to a log file
in
<filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem>
</varlistentry>
<varlistentry id="opt-max-jobs"><term><option>--max-jobs</option></term>
<term><option>-j</option></term>
<listitem><para>Sets the maximum number of build jobs that Nix will
perform in parallel to the specified number. The default is 1. A
higher value is useful on SMP systems or to exploit I/O latency.</para></listitem>
</varlistentry>
<varlistentry><term><option>--keep-going</option></term>
<term><option>-k</option></term>
<listitem><para>Keep going in case of failed builds, to the
greatest extent possible. That is, if building an input of some
derivation fails, Nix will still build the other inputs, but not the
derivation itself. Without this option, Nix stops if any build
fails (except for builds of substitutes), possibly killing builds in
progress (in case of parallel or distributed builds).</para></listitem>
</varlistentry>
<varlistentry><term><option>--keep-failed</option></term>
<term><option>-K</option></term>
<listitem><para>Specifies that in case of a build failure, the
temporary directory (usually in <filename>/tmp</filename>) in which
the build takes place should not be deleted. The path of the build
directory is printed as an informational message.
<para>
Prints out a summary of the command syntax and exits.
</para>
</listitem>
</varlistentry>
<varlistentry><term><option>--fallback</option></term>
<varlistentry>
<term><option>--version</option></term>
<listitem>
<para>
Prints out the Nix version number on standard output and exits.
</para>
</listitem>
</varlistentry>
<para>Whenever Nix attempts to build a derivation for which
substitutes are known for each output path, but realising the output
paths through the substitutes fails, fall back on building the
derivation.</para>
<para>The most common scenario in which this is useful is when we
have registered substitutes in order to perform binary distribution
from, say, a network repository. If the repository is down, the
realisation of the derivation will fail. When this option is
specified, Nix will build the derivation instead. Thus,
installation from binaries falls back on nstallation from source.
This option is not the default since it is generally not desirable
for a transient failure in obtaining the substitutes to lead to a
full build from source (with the related consumption of
resources).</para>
<varlistentry>
<term><option>--verbose</option> / <option>-v</option></term>
<listitem>
<para>
Increases the level of verbosity of diagnostic messages printed
on standard error. For each Nix operation, the information
printed on standard output is well-defined; any diagnostic
information is printed on standard error, never on standard
output.
</para>
<para>
This option may be specified repeatedly. Currently, the
following verbosity levels exist:
</para>
<variablelist>
<varlistentry>
<term>0</term>
<listitem>
<para>
<quote>Errors only</quote>: only print messages explaining
why the Nix invocation failed.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>1</term>
<listitem>
<para>
<quote>Informational</quote>: print
<emphasis>useful</emphasis> messages about what Nix is
doing. This is the default.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>2</term>
<listitem>
<para>
<quote>Talkative</quote>: print more informational messages.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>3</term>
<listitem>
<para>
<quote>Chatty</quote>: print even more informational messages.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>4</term>
<listitem>
<para>
<quote>Debug</quote>: print debug information:
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>5</term>
<listitem>
<para>
<quote>Vomit</quote>: print vast amounts of debug
information.
</para>
</listitem>
</varlistentry>
</variablelist>
</listitem>
</varlistentry>
<varlistentry><term><option>--readonly-mode</option></term>
<listitem><para>When this option is used, no attempt is made to open
the Nix database. Most Nix operations do need database access, so
those operations will fail.</para></listitem>
</varlistentry>
<varlistentry id="opt-log-type"><term><option>--log-type</option>
<replaceable>type</replaceable></term>
<varlistentry>
<term><option>--no-build-output</option> / <option>-Q</option></term>
<listitem>
<para>This option determines how the output written to standard
error is formatted. Nix’s diagnostic messages are typically
<emphasis>nested</emphasis>. For instance, when tracing Nix
expression evaluation (<command>nix-env -vvvvv</command>, messages
from subexpressions are nested inside their parent expressions. Nix
builder output is also often nested. For instance, the Nix Packages
generic builder nests the various build tasks (unpack, configure,
compile, etc.), and the GNU Make in <literal>stdenv-linux</literal>
has been patched to provide nesting for recursive Make
invocations.</para>
<para><replaceable>type</replaceable> can be one of the
following:
<variablelist>
<varlistentry><term><literal>pretty</literal></term>
<listitem><para>Pretty-print the output, indicating different
nesting levels using spaces. This is the
default.</para></listitem>
</varlistentry>
<varlistentry><term><literal>escapes</literal></term>
<listitem><para>Indicate nesting using escape codes that can be
interpreted by the <command>log2xml</command> tool in the Nix
source distribution. The resulting XML file can be fed into the
<command>log2html.xsl</command> stylesheet to create an HTML
file that can be browsed interactively, using Javascript to
expand and collapse parts of the output.</para></listitem>
</varlistentry>
<varlistentry><term><literal>flat</literal></term>
<listitem><para>Remove all nesting.</para></listitem>
</varlistentry>
</variablelist>
</para>
<para>
By default, output written by builders to standard output and
standard error is echoed to the Nix command's standard error.
This option suppresses this behaviour. Note that the builder's
standard output and error are always written to a log file in
<filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.
</para>
</listitem>
</varlistentry>
</variablelist>
<varlistentry>
<term><option>--max-jobs</option> / <option>-j</option></term>
<listitem>
<para>
Sets the maximum number of build jobs that Nix will perform in
parallel to the specified number. The default is 1. A higher
value is useful on SMP systems or to exploit I/O latency.
</para>
</listitem>
</varlistentry>
</sect1>
<varlistentry>
<term><option>--keep-going</option> / <option>-k</option></term>
<listitem>
<para>
Keep going in case of failed builds, to the greatest extent
possible. That is, if building an input of some derivation
fails, Nix will still build the other inputs, but not the
derivation itself. Without this option, Nix stops if any build
fails (except for builds of substitutes), possibly killing
builds in progress (in case of parallel or distributed builds).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--keep-failed</option> / <option>-K</option></term>
<listitem>
<para>
Specifies that in case of a build failure, the temporary
directory (usually in <filename>/tmp</filename>) in which the
build takes place should not be deleted. The path of the build
directory is printed as an informational message.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--fallback</option></term>
<listitem>
<para>
Whenever Nix attempts to realise a derivation for which a
closure is already known, but this closure cannot be realised,
fall back on normalising the derivation.
</para>
<para>
The most common scenario in which this is useful is when we have
registered substitutes in order to perform binary distribution
from, say, a network repository. If the repository is down, the
realisation of the derivation will fail. When this option is
specified, Nix will build the derivation instead. Thus,
binary installation falls back on a source installation. This
option is not the default since it is generally not desirable
for a transient failure in obtaining the substitutes to lead to
a full build from source (with the related consumption of
resources).
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>--readonly-mode</option></term>
<listitem>
<para>
When this option is used, no attempt is made to open the Nix
database. Most Nix operations do need database access, so those
operations will fail.
</para>
</listitem>
</varlistentry>
+86 -93
View File
@@ -2,24 +2,24 @@
<para>This chapter discusses how to do package management with Nix,
i.e., how to obtain, install, upgrade, and erase components. This is
the “user’s” perspective of the Nix system — people
the <quote>user's</quote> perspective of the Nix system — people
who want to <emphasis>create</emphasis> components should consult
<xref linkend='chap-writing-nix-expressions' />.</para>
<sect1><title>Basic package management</title>
<para>The main command for package management is <link
linkend="sec-nix-env"><command>nix-env</command></link>. You can use
it to install, upgrade, and erase components, and to query what
components are installed or are available for installation.</para>
<para>The main command for package management is
<command>nix-env</command>. You can use it to install, upgrade, and
erase components, and to query what components are installed or are
available for installation.</para>
<para>In Nix, different users can have different “views”
<para>In Nix, different users can have different <quote>views</quote>
on the set of installed applications. That is, there might be lots of
applications present on the system (possibly in many different
versions), but users can have a specific selection of those active —
where “active” just means that it appears in a directory
in the user’s <envar>PATH</envar>. Such a view on the set of
where <quote>active</quote> just means that it appears in a directory
in the user's <envar>PATH</envar>. Such a view on the set of
installed applications is called a <emphasis>user
environment</emphasis>, which is just a directory tree consisting of
symlinks to the files of the active applications. </para>
@@ -31,9 +31,11 @@ Nix expressions called the Nix Package collection that contains
components ranging from basic development stuff such as GCC and Glibc,
to end-user applications like Mozilla Firefox. (Nix is however not
tied to the Nix Package collection; you could write your own Nix
expressions based on it, or completely new ones.) You can download
the latest version from <ulink
url='http://catamaran.labs.cs.uu.nl/dist/nix' />.</para>
expression based on it, or completely new ones.) You can download the
latest version from <ulink
url='http://catamaran.labs.cs.uu.nl/dist/nix' />. You probably want
the latest unstable release; currently the stable releases tend to lag
behind quite a bit.</para>
<para>Assuming that you have downloaded and unpacked a release of Nix
Packages, you can view the set of available components in the release:
@@ -50,7 +52,7 @@ bzip2-1.0.2
...</screen>
where <literal>nixpkgs-<replaceable>version</replaceable></literal> is
where you’ve unpacked the release.</para>
where you've unpacked the release.</para>
<para>It is also possible to see the <emphasis>status</emphasis> of
available components, i.e., whether they are installed into the user
@@ -70,7 +72,7 @@ component is installed in your current user environment. The second
(in which case installing it into your user environment would be a
very quick operation). The last one (<literal>S</literal>) indicates
whether there is a so-called <emphasis>substitute</emphasis> for the
component, which is Nix’s mechanism for doing binary deployment. It
component, which is Nix's mechanism for doing binary deployment. It
just means that Nix know that it can fetch a pre-built component from
somewhere (typically a network server) instead of building it
locally.</para>
@@ -96,7 +98,7 @@ available somewhere. This is done using the
<command>nix-pull</command> command, which must be supplied with a URL
containing a <emphasis>manifest</emphasis> describing what binaries
are available. This URL should correspond to the Nix Packages release
that you’re using. For instance, if you obtained a release from
that you're using. For instance, if you obtained a release from
<ulink
url='http://catamaran.labs.cs.uu.nl/dist/nix/nixpkgs-0.6pre1554/' />,
then you should do:
@@ -109,7 +111,7 @@ downloading binaries from <systemitem
class='fqdomainname'>catamaran.labs.cs.uu.nl</systemitem>, instead of
building them from source. This might still take a while since all
dependencies must be downloaded, but on a reasonably fast connection
such as an DSL line it’s on the order of a few minutes.</para>
such as an DSL line it's on the order of a few minutes.</para>
<para>Naturally, packages can also be uninstalled:
@@ -125,9 +127,9 @@ release of Nix Packages, you can do:
$ nix-env -f nixpkgs-<replaceable>version</replaceable> -u subversion</screen>
This will <emphasis>only</emphasis> upgrade Subversion if there is a
“newer” version in the new set of Nix expressions, as
<quote>newer</quote> version in the new set of Nix expressions, as
defined by some pretty arbitrary rules regarding ordering of version
numbers (which generally do what you’d expect of them). To just
numbers (which generally do what you'd expect of them). To just
unconditionally replace Subversion with whatever version is in the Nix
expressions, use <parameter>-i</parameter> instead of
<parameter>-u</parameter>; <parameter>-i</parameter> will remove
@@ -141,7 +143,7 @@ $ nix-env -f nixpkgs-<replaceable>version</replaceable> -u '*'</screen>
</para>
<para>Sometimes it’s useful to be able to ask what
<para>Sometimes it's useful to be able to ask what
<command>nix-env</command> would do, without actually doing it. For
instance, to find out what packages would be upgraded by
<literal>nix-env -u '*'</literal>, you can do
@@ -173,28 +175,30 @@ set.</para></footnote></para>
</sect1>
<sect1 id="sec-profiles"><title>Profiles</title>
<sect1><title>Profiles</title>
<para>Profiles and user environments are Nix’s mechanism for
<para>Profiles and user environments are Nix's mechanism for
implementing the ability to allow differens users to have different
configurations, and to do atomic upgrades and rollbacks. To
understand how they work, it’s useful to know a bit about how Nix
understand how they work, it's useful to know a bit about how Nix
works. In Nix, components are stored in unique locations in the
<emphasis>Nix store</emphasis> (typically,
<filename>/nix/store</filename>). For instance, a particular version
of the Subversion component might be stored in a directory
<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>,
<filename>/nix/store/eeeeaf42e56b...-subversion-0.32.1/</filename>,
while another version might be stored in
<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>.
The long strings prefixed to the directory names are cryptographic
hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in
a base-32 notation, to be precise.</para></footnote> of
<emphasis>all</emphasis> inputs involved in building the component —
sources, dependencies, compiler flags, and so on. So if two
components differ in any way, they end up in different locations in
the file system, so they don’t interfere with each other. <xref
linkend='fig-user-environments' /> shows a part of a typical Nix
store.</para>
<filename>/nix/store/58823d558a6a...-subversion-0.34/</filename>. The
long hexadecimal numbers prefixed to the directory names are
cryptographic hashes<footnote><para>128 bit MD5 hashes, to be
precise.</para></footnote> of <emphasis>all</emphasis> inputs involved
in building the component — sources, dependencies, compiler flags, and
so on. So if two components differ in any way, they end up in
different locations in the file system, so they don't interfere with
each other. <xref linkend='fig-user-environments'
/><footnote><para>TODO: the figure isn't entirely up to date. It
should show multiple profiles and
<filename>~/.nix-profile</filename>.</para></footnote> shows a part of
a typical Nix store.</para>
<figure id='fig-user-environments'><title>User environments</title>
<mediaobject>
@@ -204,42 +208,41 @@ store.</para>
</mediaobject>
</figure>
<para>Of course, you wouldn’t want to type
<para>Of course, you wouldn't want to type
<screen>
$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen>
$ /nix/store/eeeeaf42e56b...-subversion-0.32.1/bin/svn</screen>
every time you want to run Subversion. Of course we could set up the
<envar>PATH</envar> environment variable to include the
<filename>bin</filename> directory of every component we want to use,
but this is not very convenient since changing <envar>PATH</envar>
doesn’t take effect for already existing processes. The solution Nix
doesn't take effect for already existing processes. The solution Nix
uses is to create directory trees of symlinks to
<emphasis>activated</emphasis> components. These are called
<emphasis>user environments</emphasis> and they are components
themselves (though automatically generated by
<command>nix-env</command>), so they too reside in the Nix store. For
instance, in <xref linkend='fig-user-environments' /> the user
environment <filename>/nix/store/5mq2jcn36ldl...-user-env</filename>
contains a symlink to just Subversion 1.1.2 (arrows in the figure
environment <filename>/nix/store/068150f63831...-user-env</filename>
contains a symlink to just Subversion 0.32.1 (arrows in the figure
indicate symlinks). This would be what we would obtain if we had done
<screen>
$ nix-env -i subversion</screen>
on a set of Nix expressions that contained Subversion 1.1.2.</para>
on a set of Nix expressions that contained Subversion 0.32.1.</para>
<para>This doesn’t in itself solve the problem, of course; you
wouldn’t want to type
<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename>
either. That’s why there are symlinks outside of the store that point
<para>This doesn't in itself solve the problem, of course; you
wouldn't want to type
<filename>/nix/store/068150f63831...-user-env/bin/svn</filename>
either. Therefore there are symlinks outside of the store that point
to the user environments in the store; for instance, the symlinks
<filename>default-42-link</filename> and
<filename>default-43-link</filename> in the example. These are called
<emphasis>generations</emphasis> since every time you perform a
<command>nix-env</command> operation, a new user environment is
generated based on the current one. For instance, generation 43 was
created from generation 42 when we did
<filename>42</filename> and <filename>43</filename> in the example.
These are called <emphasis>generations</emphasis> since every time you
perform a <command>nix-env</command> operation, a new user environment
is generated based on the current one. For instance, generation 43
was created from generation 42 when we did
<screen>
$ nix-env -i subversion mozilla</screen>
@@ -248,14 +251,14 @@ on a set of Nix expressions that contained Mozilla and a new version
of Subversion.</para>
<para>Generations are grouped together into
<emphasis>profiles</emphasis> so that different users don’t interfere
with each other if they don’t want to. For example:
<emphasis>profiles</emphasis> so that different users don't interfere
with each other if they don't want to. For example:
<screen>
$ ls -l /nix/var/nix/profiles/
...
lrwxrwxrwx 1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env
lrwxrwxrwx 1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env
lrwxrwxrwx 1 eelco ... default-42-link -> /nix/store/068150f63831...-user-env
lrwxrwxrwx 1 eelco ... default-43-link -> /nix/store/84c85f89ddbf...-user-env
lrwxrwxrwx 1 eelco ... default -> default-43-link</screen>
This shows a profile called <filename>default</filename>. The file
@@ -265,7 +268,7 @@ operation, a new user environment and generation link are created
based on the current one, and finally the <filename>default</filename>
symlink is made to point at the new generation. This last step is
atomic on Unix, which explains how we can do atomic upgrades. (Note
that the building/installing of new components doesn’t interfere in
that the building/installing of new components doesn't interfere in
any way with old components, since they are stored in different
locations in the Nix store.)</para>
@@ -290,13 +293,13 @@ can also see all available generations:
$ nix-env --list-generations</screen></para>
<para>Actually, there is another level of indirection not shown in the
figure above. You generally wouldn’t have
figure above. You generally wouldn't have
<filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename>
in your <envar>PATH</envar>. Rather, there is a symlink
<filename>~/.nix-profile</filename> that points to your current
profile. This means that you should put
<filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar>
(and indeed, that’s what the initialisation script
(and indeed, that's what the initialisation script
<filename>/nix/etc/profile.d/nix.sh</filename> does). This makes it
easier to switch to a different profile. You can do that using the
command <command>nix-env --switch-profile</command>:
@@ -307,7 +310,7 @@ $ nix-env --switch-profile /nix/var/nix/profiles/my-profile
$ nix-env --switch-profile /nix/var/nix/profiles/default</screen>
These commands switch to the <filename>my-profile</filename> and
default profile, respectively. If the profile doesn’t exist, it will
default profile, respectively. If the profile doesn't exist, it will
be created automatically. You should be careful about storing a
profile in another location than the <filename>profiles</filename>
directory, since otherwise it might not be used as a root of the
@@ -334,7 +337,7 @@ This will <emphasis>not</emphasis> change the
(<option>-u</option>) and uninstall (<option>-e</option>) never
actually delete components from the system. All they do (as shown
above) is to create a new user environment that no longer contains
symlinks to the “deleted” components.</para>
symlinks to the <quote>deleted</quote> components.</para>
<para>Of course, since disk space is not infinite, unused components
should be removed at some point. You can do this by running the Nix
@@ -343,7 +346,7 @@ not used (directly or indirectly) by any generation of any
profile.</para>
<para>Note however that as long as old generations reference a
component, it will not be deleted. After all, we wouldn’t be able to
component, it will not be deleted. After all, we wouldn't be able to
do a rollback otherwise. So in order for garbage collection to be
effective, you should also delete (some) old generations. Of course,
this should only be done if you are certain that you will not need to
@@ -367,61 +370,51 @@ $ nix-env --delete-generations 10 11 14</screen>
garbage collector as follows:
<screen>
$ nix-store --gc</screen>
$ nix-collect-garbage</screen>
If you are feeling uncertain, you can also first view what files would
be deleted:
You can alo first view what files would be deleted:
<screen>
$ nix-store --gc --print-dead</screen>
$ nix-collect-garbage --print-dead</screen>
Likewise, the option <option>--print-live</option> will show the paths
that <emphasis>won’t</emphasis> be deleted.</para>
that <emphasis>won't</emphasis> be deleted.</para>
<sect2><title>Garbage collector roots</title>
<sect2 id="ssec-gc-roots"><title>Garbage collector roots</title>
<para>TODO</para>
<para>The roots of the garbage collector are all store paths to which
there are symlinks in the directory
<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>.
For instance, the following command makes the path
<filename>/nix/store/d718ef...-foo</filename> a root of the collector:
<screen>
$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen>
That is, after this command, the garbage collector will not remove
<filename>/nix/store/d718ef...-foo</filename> or any of its
dependencies.</para>
<para>Subdirectories of
<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>
are also searched for symlinks. Symlinks to non-store paths are
followed and searched for roots, but symlinks to non-store paths
<emphasis>inside</emphasis> the paths reached in that way are not
followed to prevent infinite recursion.</para>
<para>The garbage collector uses as roots all store expressions
mentioned in all files with extension <filename>.gcroot</filename> in
the directory
<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>,
or in any file or directory symlinked to from that directory. E.g.,
by default,
<filename><replaceable>prefix</replaceable>/var/nix/gcroots/</filename>
contains a symlink to
<filename><replaceable>prefix</replaceable>/var/nix/profiles/</filename>,
so all generations of all profiles are also roots of the collector.</para>
</sect2>
</sect1>
<sect1 id="sec-channels"><title>Channels</title>
<sect1><title>Channels</title>
<para>If you want to stay up to date with a set of packages, it’s not
<para>If you want to stay up to date with a set of packages, it's not
very convenient to manually download the latest set of Nix expressions
for those packages, use <command>nix-pull</command> to register
pre-built binaries (if available), and upgrade using
<command>nix-env</command>. Fortunately, there’s a better way:
<command>nix-env</command>. Fortunately, there's a better way:
<emphasis>Nix channels</emphasis>.</para>
<para>A Nix channel is just a URL that points to a place that contains
a set of Nix expressions and a manifest. Using the command <link
linkend="sec-nix-channel"><command>nix-channel</command></link> you
can automatically stay up to date with whatever is available at that
URL.</para>
a set of Nix expressions and a manifest. Using the command
<command>nix-channel</command> you can automatically stay up to date
with whatever is available at that URL.</para>
<para>You can “subscribe” to a channel using
<para>You can <quote>subscribe</quote> to a channel using
<command>nix-channel --add</command>, e.g.,
<screen>
@@ -434,7 +427,7 @@ of the Nix Packages collection. (Instead of
stability, but right now is just outdated.) Subscribing really just
means that the URL is added to the file
<filename>~/.nix-channels</filename>. Right now there is no command
to “unsubscribe”; you should just edit that file manually
to <quote>unsubscribe</quote>; you should just edit that file manually
and delete the offending URL.</para>
<para>To obtain the latest Nix expressions available in a channel, do
@@ -447,7 +440,7 @@ This downloads the Nix expressions in every channel (downloaded from
and registers any available pre-built binaries in every channel
(by <command>nix-pull</command>ing
<literal><replaceable>url</replaceable>/MANIFEST</literal>). It also
makes the union of each channel’s Nix expressions the default for
makes the union of each channel's Nix expressions the default for
<command>nix-env</command> operations. Consequently, you can then say
<screen>
+7 -7
View File
@@ -88,8 +88,8 @@ $ nix-channel --update
$ nix-env -u '*'</screen>
The latter command will upgrade each installed component for which
there is a “newer” version (as determined by comparing the version
numbers).</para></listitem>
there is a <quote>newer</quote> version (as determined by comparing
the version numbers).</para></listitem>
<listitem><para>If you're unhappy with the result of a
<command>nix-env</command> action (e.g., an upgraded component turned
@@ -106,12 +106,12 @@ actually delete them:
<screen>
$ nix-env --delete-generations old
$ nix-store --gc</screen>
$ nix-collect-garbage</screen>
The first command deletes old “generations” of your profile (making
rollbacks impossible, but also making the components in those old
generations available for garbage collection), while the second
command actually deletes them.</para></listitem>
The first command deletes old <quote>generations</quote> of your
profile (making rollbacks impossible, but also making the components
in those old generations available for garbage collection), while the
second command actually deletes them.</para></listitem>
</orderedlist>
+2 -4
View File
@@ -102,8 +102,6 @@ pre.screen
.note,.warning
{
margin-top: 1em;
margin-bottom: 1em;
border: 1px solid #6185a0;
padding: 0px 1em;
background: #fffff5;
@@ -156,7 +154,7 @@ a:hover { background: #ffffcd; }
Special elements:
***************************************************************************/
tt, code
tt
{
color: #400000;
}
@@ -231,4 +229,4 @@ div.epigraph
table.productionset table.productionset
{
font-family: monospace;
}
}
+12 -69
View File
@@ -1,71 +1,14 @@
<appendix><title>Troubleshooting</title>
<para>This section provides solutions for some common problems.</para>
<sect1><title>Berkeley DB: <quote>Cannot allocate memory</quote></title>
<para>Symptom: Nix operations (in particular the
<command>nix-store</command> operations <option>--gc</option>,
<option>--verify</option>, and <option>--clear-substitutes</option> —
the latter being called by <command>nix-channel --update</command>)
failing:
<screen>
$ nix-store --verify
error: Db::del: Cannot allocate memory</screen>
</para>
<para>Possible solution: make sure that no Nix processes are running,
then do:
<screen>
$ cd /nix/var/nix/db
$ rm __db.00*</screen>
</para>
</sect1>
<sect1><title>Collisions in <command>nix-env</command></title>
<para>Symptom: when installing or upgrading, you get an error message such as
<screen>
$ nix-env -i docbook-xml
...
adding /nix/store/s5hyxgm62gk2...-docbook-xml-4.2
collission between `/nix/store/s5hyxgm62gk2...-docbook-xml-4.2/xml/dtd/docbook/calstblx.dtd'
and `/nix/store/06h377hr4b33...-docbook-xml-4.3/xml/dtd/docbook/calstblx.dtd'
at /nix/store/...-builder.pl line 62.</screen>
</para>
<para>The cause is that two installed packages in the user environment
have overlapping filenames (e.g.,
<filename>xml/dtd/docbook/calstblx.dtd</filename>. This usually
happens when you accidentally try to install two versions of the same
package. For instance, in the example above, the Nix Packages
collection contains two versions of <literal>docbook-xml</literal>, so
<command>nix-env -i</command> will try to install both. The default
user environment builder has no way to way to resolve such conflicts,
so it just gives up.</para>
<para>Solution: remove one of the offending packages from the user
environment (if already installed) using <command>nix-env
-u</command>, or specify exactly which version should be installed
(e.g., <literal>nix-env -i docbook-xml-4.2</literal>).</para>
<para>Alternatively, you can modify the user environment builder
script (in
<filename><replaceable>prefix</replaceable>/share/nix/corepkgs/buildenv/builder.pl</filename>)
to implement some conflict resolution policy. E.g., the script could
be modified to rename conflicting file names, or to pick one over the
other.</para>
</sect1>
<appendix>
<title>Troubleshooting</title>
<para>
(Nothing.)
</para>
</appendix>
<!--
local variables:
sgml-parent-document: ("book.xml" "appendix")
end:
-->
+7 -19
View File
@@ -383,7 +383,7 @@ some fragments of
that can be built by Nix (since when we fill in the arguments of
the function, what we get is its body, which is the call to
<varname>stdenv.mkDerivation</varname> in <xref
linkend='ex-hello-nix' />).</para>
linkend='ex-hello-nix ' />).</para>
</callout>
@@ -447,8 +447,7 @@ following:
(import pkgs/system/i686-linux.nix).hello</programlisting>
Call it <filename>test.nix</filename>. You can then build it without
installing it using the command <link
linkend="sec-nix-build"><command>nix-build</command></link>:
installing it using the command <command>nix-build</command>:
<screen>
$ nix-build ./test.nix
@@ -576,7 +575,7 @@ builder becomes even shorter:
<programlisting>
. $stdenv/setup
genericBuild</programlisting>
genericBuilder</programlisting>
In fact, <varname>mkDerivation</varname> provides a default builder
that looks exactly like that, so it is actually possible to omit the
@@ -818,7 +817,7 @@ set.</para>
</simplesect>
<simplesect id="ss-functions"><title>Functions</title>
<simplesect><title>Functions</title>
<para>Functions have the following form:
@@ -1104,7 +1103,7 @@ weakest binding).</para>
</simplesect>
<simplesect id="ssec-derivation"><title>Derivations</title>
<simplesect><title>Derivations</title>
<para>The most important built-in function is
<function>derivation</function>, which is used to describe a
@@ -1113,7 +1112,7 @@ set, the attributes of which specify the inputs of the build.</para>
<itemizedlist>
<listitem id="attr-system"><para>There must be an attribute named
<listitem><para>There must be an attribute named
<varname>system</varname> whose value must be a string specifying a
Nix platform identifier, such as <literal>"i686-linux"</literal> or
<literal>"powerpc-darwin"</literal><footnote><para>To figure out
@@ -1254,17 +1253,6 @@ command-line argument. See <xref linkend='sec-standard-environment'
performed by looking for the hash parts of file names of the
inputs.</para></listitem>
<listitem><para>After the build, Nix sets the last-modified
timestamp on all files in the build result to 0 (00:00:00 1/1/1970
UTC), sets the group to the default group, and sets the mode of the
file to 0444 or 0555 (i.e., read-only, with execute permission
enabled if the file was originally executable). Note that possible
<literal>setuid</literal> and <literal>setgid</literal> bits are
cleared. Setuid and setgid programs are not currently supported by
Nix. This is because the Nix archives used in deployment have no
concept of ownership information, and because it makes the build
result dependent on the user performing the build.</para></listitem>
</itemizedlist>
</para>
@@ -1383,7 +1371,7 @@ following:
</para>
<para>The <filename>setup</filename> script also exports a function
called <function>genericBuild</function> that knows how to build
called <function>genericBuilder</function> that knows how to build
typical Autoconf-style components. It can be customised to perform
builds for any type of component. It is advisable to use
<function>genericBuild</function> since it provides facilities that
+3 -3
View File
@@ -34,12 +34,12 @@ endif
# CWI ATerm
ATERM = aterm-2.3.1
ATERM = aterm-2.2
$(ATERM).tar.gz:
@echo "Nix requires the CWI ATerm library to build."
@echo "Please download version 2.3.1 from"
@echo " http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.3.1.tar.gz"
@echo "Please download version 2.2 from"
@echo " http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.2.tar.gz"
@echo "and place it in the externals/ directory."
false
-49
View File
@@ -1,49 +0,0 @@
### Option `gc-keep-outputs'
#
# If `true', the garbage collector will keep the outputs of
# non-garbage derivations. If `false' (default), outputs will be
# deleted unless they are GC roots themselves (or reachable from other
# roots).
#
# In general, outputs must be registered as roots separately.
# However, even if the output of a derivation is registered as a root,
# the collector will still delete store paths that are used only at
# build time (e.g., the C compiler, or source tarballs downloaded from
# the network). To prevent it from doing so, set this option to
# `true'.
gc-keep-outputs = false
### Option `gc-keep-derivations'
#
# If `true' (default), the garbage collector will keep the derivations
# from which non-garbage store paths were built. If `false', they
# will be deleted unless explicitly registered as a root (or reachable
# from other roots).
#
# Keeping derivation around is useful for querying and traceability
# (e.g., it allows you to ask with what dependencies or options a
# store path was built), so by default this option is on. Turn it off
# to safe a bit of disk space (or a lot if `gc-keep-outputs' is also
# turned on).
gc-keep-derivations = true
### Option `env-keep-derivations'
#
# If `false' (default), derivations are not stored in Nix user
# environments. That is, the derivation any build-time-only
# dependencies may be garbage-collected.
#
# If `true', when you add a Nix derivation to a user environment, the
# path of the derivation is stored in the user environment. Thus, the
# derivation will not be garbage-collected until the user environment
# generation is deleted (`nix-env --delete-generations'). To prevent
# build-time-only dependencies from being collected, you should also
# turn on `gc-keep-outputs'.
#
# The difference between this option and `gc-keep-derivations' is that
# this one is `sticky': it applies to any user environment created
# while this option was enabled, while `gc-keep-derivations' only
# applies at the moment the garbage collector is run.
env-keep-derivations = false
+7 -8
View File
@@ -2,17 +2,18 @@ bin_SCRIPTS = nix-collect-garbage \
nix-pull nix-push nix-prefetch-url \
nix-install-package nix-channel nix-build
noinst_SCRIPTS = nix-profile.sh generate-patches.pl
noinst_SCRIPTS = nix-profile.sh
nix-pull nix-push: readmanifest.pm download-using-manifests.pl
nix-pull nix-push: readmanifest.pm
install-exec-local: readmanifest.pm download-using-manifests.pl
install-exec-local: readmanifest.pm
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
# !!! don't overwrite local modifications
$(INSTALL_DATA) prebuilts.conf $(DESTDIR)$(sysconfdir)/nix/prebuilts.conf
include ../substitute.mk
@@ -20,7 +21,5 @@ EXTRA_DIST = nix-collect-garbage.in \
nix-pull.in nix-push.in nix-profile.sh.in \
nix-prefetch-url.in nix-install-package.in \
nix-channel.in \
readmanifest.pm.in \
nix-build.in \
download-using-manifests.pl.in \
generate-patches.pl.in
prebuilts.conf readmanifest.pm.in \
nix-build.in
-111
View File
@@ -1,111 +0,0 @@
#! /usr/bin/perl -w
use strict;
my @paths = `nix-store -qR /home/eelco/.nix-profile/bin/firefox`;
my %copyMap;
my %rewriteMap;
my $counter = 0;
foreach my $path (@paths) {
chomp $path;
$path =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$path'";
my $hash = $2;
# my $newHash = "deadbeef" . (sprintf "%024d", $counter++);
my $newHash = "deadbeef" . substr($hash, 0, 24);
my $newPath = "/home/eelco/chroot/$1/$newHash-$3";
die unless length $newHash == length $hash;
$copyMap{$path} = $newPath;
$rewriteMap{$hash} = $newHash;
}
my %rewriteMap2;
sub rewrite;
sub rewrite {
my $src = shift;
my $dst = shift;
if (-l $dst) {
my $target = readlink $dst or die;
foreach my $srcHash (keys %rewriteMap2) {
my $dstHash = $rewriteMap{$srcHash};
print " $srcHash -> $dstHash\n";
$target =~ s/$srcHash/$dstHash/g;
}
unlink $dst or die;
symlink $target, $dst;
}
elsif (-f $dst) {
print "$dst\n";
foreach my $srcHash (keys %rewriteMap2) {
my $dstHash = $rewriteMap{$srcHash};
print " $srcHash -> $dstHash\n";
my @stats = lstat $dst or die;
system "sed s/$srcHash/$dstHash/g < '$dst' > '$dst.tmp'";
die if $? != 0;
rename "$dst.tmp", $dst or die;
chmod $stats[2], $dst or die;
}
}
elsif (-d $dst) {
chmod 0755, $dst;
opendir(DIR, "$dst") or die "cannot open `$dst': $!";
my @files = readdir DIR;
closedir DIR;
foreach my $file (@files) {
next if $file eq "." || $file eq "..";
rewrite "$src/$file", "$dst/$file";
}
}
}
foreach my $src (keys %copyMap) {
my $dst = $copyMap{$src};
print "$src -> $dst\n";
if (!-e $dst) {
system "cp -prd $src $dst";
die if $? != 0;
my @refs = `nix-store -q --references $src`;
%rewriteMap2 = ();
foreach my $ref (@refs) {
chomp $ref;
$ref =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$ref'";
my $hash = $2;
$rewriteMap2{$hash} = $rewriteMap{$hash};
}
rewrite $src, $dst;
}
}
-263
View File
@@ -1,263 +0,0 @@
#! @perl@ -w -I@libexecdir@/nix
use strict;
use readmanifest;
my $manifestDir = "@localstatedir@/nix/manifests";
my $logFile = "@localstatedir@/log/nix/downloads";
open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
# Check the arguments.
die unless scalar @ARGV == 1;
my $targetPath = $ARGV[0];
my $date = `date` or die;
chomp $date;
print LOGFILE "$$ get $targetPath $date\n";
print "\n*** Trying to download/patch `$targetPath'\n";
# Load all manifests.
my %narFiles;
my %patches;
my %successors;
for my $manifest (glob "$manifestDir/*.nixmanifest") {
# print STDERR "reading $manifest\n";
if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
print STDERR "you have an old-style manifest `$manifest'; please delete it\n";
exit 1;
}
}
# Build a graph of all store paths that might contribute to the
# construction of $targetPath, and the special node "start". The
# edges are either patch operations, or downloads of full NAR files.
# The latter edges only occur between "start" and a store path.
my %graph;
$graph{"start"} = {d => 0, pred => undef, edges => []};
my @queue = ();
my $queueFront = 0;
my %done;
sub addToQueue {
my $v = shift;
return if defined $done{$v};
$done{$v} = 1;
push @queue, $v;
}
sub addNode {
my $u = shift;
$graph{$u} = {d => 999999999999, pred => undef, edges => []}
unless defined $graph{$u};
}
sub addEdge {
my $u = shift;
my $v = shift;
my $w = shift;
my $type = shift;
my $info = shift;
addNode $u;
push @{$graph{$u}->{edges}},
{weight => $w, start => $u, end => $v, type => $type, info => $info};
my $n = scalar @{$graph{$u}->{edges}};
}
addToQueue $targetPath;
sub isValidPath {
my $p = shift;
system "@bindir@/nix-store --check-validity '$p' 2> /dev/null";
return $? == 0;
}
sub parseHash {
my $hash = shift;
if ($hash =~ /^(.+):(.+)$/) {
return ($1, $2);
} else {
return ("md5", $hash);
}
}
while ($queueFront < scalar @queue) {
my $u = $queue[$queueFront++];
# print "$u\n";
addNode $u;
# If the path already exists, it has distance 0 from the "start"
# node.
if (isValidPath($u)) {
addEdge "start", $u, 0, "present", undef;
}
else {
# Add patch edges.
my $patchList = $patches{$u};
foreach my $patch (@{$patchList}) {
if (isValidPath($patch->{basePath})) {
# !!! this should be cached
my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash};
my $format = "--base32";
$format = "" if $baseHashAlgo eq "md5";
my $hash = `@bindir@/nix-hash --type '$baseHashAlgo' $format "$patch->{basePath}"`;
chomp $hash;
# print " MY HASH is $hash\n";
if ($hash ne $baseHash) {
print LOGFILE "$$ rejecting $patch->{basePath}\n";
next;
}
}
# print " PATCH from $patch->{basePath}\n";
addToQueue $patch->{basePath};
addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
}
# Add NAR file edges to the start node.
my $narFileList = $narFiles{$u};
foreach my $narFile (@{$narFileList}) {
# print " NAR from $narFile->{url}\n";
addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
if ($u eq $targetPath) {
print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
}
}
}
}
# Run Dijkstra's shortest path algorithm to determine the shortest
# sequence of download and/or patch actions that will produce
# $targetPath.
sub byDistance { # sort by distance, reversed
return -($graph{$a}->{d} <=> $graph{$b}->{d});
}
my @todo = keys %graph;
while (scalar @todo > 0) {
# Remove the closest element from the todo list.
@todo = sort byDistance @todo;
my $u = pop @todo;
my $u_ = $graph{$u};
# print "IN $u $u_->{d}\n";
foreach my $edge (@{$u_->{edges}}) {
my $v_ = $graph{$edge->{end}};
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
$v_->{d} = $u_->{d} + $edge->{weight};
# Store the edge; to edge->start is actually the
# predecessor.
$v_->{pred} = $edge;
# print " RELAX $edge->{end} $v_->{d}\n";
}
}
}
# Retrieve the shortest path from "start" to $targetPath.
my @path = ();
my $cur = $targetPath;
die "don't know how to produce $targetPath\n"
unless defined $graph{$targetPath}->{pred};
while ($cur ne "start") {
push @path, $graph{$cur}->{pred};
$cur = $graph{$cur}->{pred}->{start};
}
# Traverse the shortest path, perform the actions described by the
# edges.
my $curStep = 1;
my $maxStep = scalar @path;
sub downloadFile {
my $url = shift;
my ($hashAlgo, $hash) = parseHash(shift);
$ENV{"PRINT_PATH"} = 1;
$ENV{"QUIET"} = 1;
$ENV{"NIX_HASH_ALGO"} = $hashAlgo;
my ($hash2, $path) = `@bindir@/nix-prefetch-url '$url' '$hash'`;
die "download of `$url' failed" unless $? == 0;
chomp $hash2;
chomp $path;
die "hash mismatch, expected $hash, got $hash2" if $hash ne $hash2;
return $path;
}
while (scalar @path > 0) {
my $edge = pop @path;
my $u = $edge->{start};
my $v = $edge->{end};
print "\n*** Step $curStep/$maxStep: ";
$curStep++;
if ($edge->{type} eq "present") {
print "using already present path `$v'\n";
print LOGFILE "$$ present $v\n";
}
elsif ($edge->{type} eq "patch") {
my $patch = $edge->{info};
print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
# Download the patch.
print " downloading patch...\n";
my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
# Turn the base path into a NAR archive, to which we can
# actually apply the patch.
print " packing base path...\n";
system "@bindir@/nix-store --dump $patch->{basePath} > /tmp/nar";
die "cannot dump `$patch->{basePath}'" if ($? != 0);
# Apply the patch.
print " applying patch...\n";
system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
# Unpack the resulting NAR archive into the target path.
print " unpacking patched archive...\n";
system "@bindir@/nix-store --restore $v < /tmp/nar2";
die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
}
elsif ($edge->{type} eq "narfile") {
my $narFile = $edge->{info};
print "downloading `$narFile->{url}' into `$v'\n";
print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
# Download the archive.
print " downloading archive...\n";
my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
# Unpack the archive into the target path.
print " unpacking archive...\n";
system "@bunzip2@ < '$narFilePath' | @bindir@/nix-store --restore '$v'";
die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
}
}
print LOGFILE "$$ success\n";
close LOGFILE;
-75
View File
@@ -1,75 +0,0 @@
#! /usr/bin/perl -w
use strict;
use readmanifest;
# Read the archive directories.
my @archives = ();
my %archives;
sub readDir {
my $dir = shift;
opendir(DIR, "$dir") or die "cannot open `$dir': $!";
my @as = readdir DIR;
foreach my $archive (@as) {
push @archives, $archive;
$archives{$archive} = "$dir/$archive";
}
closedir DIR;
}
readDir "/mnt/scratchy/eelco/public_html/nix-cache";
readDir "/mnt/scratchy/eelco/public_html/patches";
print STDERR scalar @archives, "\n";
# Read the manifests.
my %narFiles;
my %patches;
my %successors;
foreach my $manifest (@ARGV) {
print STDERR "loading $manifest\n";
if (readManifest($manifest, \%narFiles, \%patches, \%successors, 1) < 3) {
# die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n";
}
}
# Find the live archives.
my %usedFiles;
foreach my $narFile (keys %narFiles) {
foreach my $file (@{$narFiles{$narFile}}) {
$file->{url} =~ /\/([^\/]+)$/;
my $basename = $1;
die unless defined $basename;
# print $basename, "\n";
$usedFiles{$basename} = 1;
die "missing archive `$basename'"
unless defined $archives{$basename};
}
}
foreach my $patch (keys %patches) {
foreach my $file (@{$patches{$patch}}) {
$file->{url} =~ /\/([^\/]+)$/;
my $basename = $1;
die unless defined $basename;
# print $basename, "\n";
$usedFiles{$basename} = 1;
die "missing archive `$basename'"
unless defined $archives{$basename};
}
}
# Print out the dead archives.
foreach my $archive (@archives) {
next if $archive eq "." || $archive eq "..";
if (!defined $usedFiles{$archive}) {
print $archives{$archive}, "\n";
}
}
-376
View File
@@ -1,376 +0,0 @@
#! @perl@ -w -I@libexecdir@/nix
use strict;
use POSIX qw(tmpnam);
use readmanifest;
die unless scalar @ARGV == 5;
my $hashAlgo = "sha256";
my $cacheDir = $ARGV[0];
my $patchesDir = $ARGV[1];
my $patchesURL = $ARGV[2];
my $srcDir = $ARGV[3];
my $dstDir = $ARGV[4];
my $tmpdir;
do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
print "TEMP = $tmpdir\n";
#END { rmdir $tmpdir; }
my %srcNarFiles;
my %srcPatches;
my %srcSuccessors;
my %dstNarFiles;
my %dstPatches;
my %dstSuccessors;
readManifest "$srcDir/MANIFEST",
\%srcNarFiles, \%srcPatches, \%srcSuccessors;
readManifest "$dstDir/MANIFEST",
\%dstNarFiles, \%dstPatches, \%dstSuccessors;
sub findOutputPaths {
my $narFiles = shift;
my $successors = shift;
my %outPaths;
foreach my $p (keys %{$narFiles}) {
# Ignore store expressions.
next if ($p =~ /\.store$/);
next if ($p =~ /\.drv$/);
# Ignore builders (too much ambiguity -- they're all called
# `builder.sh').
next if ($p =~ /\.sh$/);
next if ($p =~ /\.patch$/);
# Don't bother including tar files etc.
next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
$outPaths{$p} = 1;
}
return %outPaths;
}
print "finding src output paths...\n";
my %srcOutPaths = findOutputPaths \%srcNarFiles, \%srcSuccessors;
print "finding dst output paths...\n";
my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
sub getNameVersion {
my $p = shift;
$p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
my $name = $1;
my $version = $2;
$name =~ s/^-//;
$version =~ s/^-//;
return ($name, $version);
}
# A quick hack to get a measure of the `distance' between two
# versions: it's just the position of the first character that differs
# (or 999 if they are the same).
sub versionDiff {
my $s = shift;
my $t = shift;
my $i;
return 999 if $s eq $t;
for ($i = 0; $i < length $s; $i++) {
return $i if $i >= length $t or
substr($s, $i, 1) ne substr($t, $i, 1);
}
return $i;
}
sub getNarBz2 {
my $narFiles = shift;
my $storePath = shift;
my $narFileList = $$narFiles{$storePath};
die "missing store expression $storePath" unless defined $narFileList;
my $narFile = @{$narFileList}[0];
die unless defined $narFile;
$narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1;
return "$cacheDir/$1";
}
sub containsPatch {
my $patches = shift;
my $storePath = shift;
my $basePath = shift;
my $patchList = $$patches{$storePath};
return 0 if !defined $patchList;
my $found = 0;
foreach my $patch (@{$patchList}) {
# !!! baseHash might differ
return 1 if $patch->{basePath} eq $basePath;
}
return 0;
}
# Compute the "weighted" number of uses of a path in the build graph.
sub computeUses {
my $narFiles = shift;
my $path = shift;
# Find the deriver of $path.
return 1 unless defined $$narFiles{$path};
my $deriver = @{$$narFiles{$path}}[0]->{deriver};
return 1 unless defined $deriver && $deriver ne "";
# print " DERIVER $deriver\n";
# Optimisation: build the referers graph from the references
# graph.
my %referers;
foreach my $q (keys %{$narFiles}) {
my @refs = split " ", @{$$narFiles{$q}}[0]->{references};
foreach my $r (@refs) {
$referers{$r} = [] unless defined $referers{$r};
push @{$referers{$r}}, $q;
}
}
# Determine the shortest path from $deriver to all other reachable
# paths in the `referers' graph.
my %dist;
$dist{$deriver} = 0;
my @queue = ($deriver);
my $pos = 0;
while ($pos < scalar @queue) {
my $p = $queue[$pos];
$pos++;
foreach my $q (@{$referers{$p}}) {
if (!defined $dist{$q}) {
$dist{$q} = $dist{$p} + 1;
# print " $q $dist{$q}\n";
push @queue, $q;
}
}
}
my $wuse = 1.0;
foreach my $user (keys %dist) {
next if $user eq $deriver;
# print " $user $dist{$user}\n";
$wuse += 1.0 / 2.0**$dist{$user};
}
# print " XXX $path $wuse\n";
return $wuse;
}
# For each output path in the destination, see if we need to / can
# create a patch.
print "creating patches...\n";
foreach my $p (keys %dstOutPaths) {
# If exactly the same path already exists in the source, skip it.
next if defined $srcOutPaths{$p};
print " $p\n";
# If not, then we should find the paths in the source that are
# `most' likely to be present on a system that wants to install
# this path.
(my $name, my $version) = getNameVersion $p;
my @closest = ();
my $closestVersion;
my $minDist = -1; # actually, larger means closer
# Find all source paths with the same name.
foreach my $q (keys %srcOutPaths) {
(my $name2, my $version2) = getNameVersion $q;
if ($name eq $name2) {
# If the sizes differ too much, then skip. This
# disambiguates between, e.g., a real component and a
# wrapper component (cf. Firefox in Nixpkgs).
my $srcSize = @{$srcNarFiles{$q}}[0]->{size};
my $dstSize = @{$dstNarFiles{$p}}[0]->{size};
my $ratio = $srcSize / $dstSize;
$ratio = 1 / $ratio if $ratio < 1;
# print " SIZE $srcSize $dstSize $ratio $q\n";
if ($ratio >= 3) {
print " SKIPPING $q due to size ratio $ratio ($srcSize $dstSize)\n";
next;
}
# If the numbers of weighted uses differ too much, then
# skip. This disambiguates between, e.g., the bootstrap
# GCC and the final GCC in Nixpkgs.
my $srcUses = computeUses \%srcNarFiles, $q;
my $dstUses = computeUses \%dstNarFiles, $p;
$ratio = $srcUses / $dstUses;
$ratio = 1 / $ratio if $ratio < 1;
print " USE $srcUses $dstUses $ratio $q\n";
if ($ratio >= 2) {
print " SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
next;
}
# If there are multiple matching names, include the ones
# with the closest version numbers.
my $dist = versionDiff $version, $version2;
if ($dist > $minDist) {
$minDist = $dist;
@closest = ($q);
$closestVersion = $version2;
} elsif ($dist == $minDist) {
push @closest, $q;
}
}
}
if (scalar(@closest) == 0) {
print " NO BASE: $p\n";
next;
}
foreach my $closest (@closest) {
# Generate a patch between $closest and $p.
print " $p <- $closest\n";
# If the patch already exists, skip it.
if (containsPatch(\%srcPatches, $p, $closest) ||
containsPatch(\%dstPatches, $p, $closest))
{
print " skipping, already exists\n";
next;
}
# next;
my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
system("@bunzip2@ < $srcNarBz2 > $tmpdir/A") == 0
or die "cannot unpack $srcNarBz2";
system("@bunzip2@ < $dstNarBz2 > $tmpdir/B") == 0
or die "cannot unpack $dstNarBz2";
system("@libexecdir@/bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
or die "cannot compute binary diff";
my $baseHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/A` or die;
chomp $baseHash;
my $narHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/B` or die;
chomp $narHash;
my $narDiffHash = `@bindir@/nix-hash --flat --type $hashAlgo --base32 $tmpdir/DIFF` or die;
chomp $narDiffHash;
my $narDiffSize = (stat "$tmpdir/DIFF")[7];
my $dstNarBz2Size = (stat $dstNarBz2)[7];
if ($narDiffSize >= $dstNarBz2Size) {
print " rejecting; patch bigger than full archive\n";
next;
}
my $finalName =
"$narDiffHash.nar-bsdiff";
print " size $narDiffSize; full size $dstNarBz2Size\n";
if (-e "$patchesDir/$finalName") {
print " not copying, already exists\n";
}
else {
system("cp '$tmpdir/DIFF' '$patchesDir/$finalName.tmp'") == 0
or die "cannot copy diff";
rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
or die "cannot rename $patchesDir/$finalName.tmp";
}
# Add the patch to the manifest.
addPatch \%dstPatches, $p,
{ url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash"
, size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash"
, narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff"
}, 0;
}
}
# Add in any potentially useful patches in the source (namely, those
# patches that produce either paths in the destination or paths that
# can be used as the base for other useful patches).
my $changed;
do {
# !!! we repeat this to reach the transitive closure; inefficient
$changed = 0;
foreach my $p (keys %srcPatches) {
my $patchList = $srcPatches{$p};
my $include = 0;
# Is path $p included in the destination? If so, include
# patches that produce it.
$include = 1 if (defined $dstNarFiles{$p});
# Is path $p a path that serves as a base for paths in the
# destination? If so, include patches that produce it.
foreach my $q (keys %dstPatches) {
foreach my $patch (@{$dstPatches{$q}}) {
# !!! check baseHash
$include = 1 if ($p eq $patch->{basePath});
}
}
if ($include) {
foreach my $patch (@{$patchList}) {
$changed = 1 if addPatch \%dstPatches, $p, $patch;
}
}
}
} while $changed;
# Rewrite the manifest of the destination (with the new patches).
writeManifest "$dstDir/MANIFEST",
\%dstNarFiles, \%dstPatches, \%dstSuccessors;
Regular → Executable
+15 -33
View File
@@ -8,52 +8,34 @@ if test -z "$nixExpr"; then
fi
extraArgs=
addDrvLink=0
addOutLink=1
noLink=
trap 'rm -f ./.nix-build-tmp-*' EXIT
# Process the arguments.
for i in "$@"; do
case "$i" in
--add-drv-link)
addDrvLink=1
;;
--no-link)
addOutLink=0
noLink=1
;;
-*)
extraArgs="$extraArgs $i"
;;
*)
# Instantiate the Nix expression.
prefix=
if test "$addDrvLink" = 0; then prefix=.nix-build-tmp-; fi
storeExprs=$(@bindir@/nix-instantiate \
--add-root ./${prefix}derivation --indirect \
"$i")
storeExprs=$(nix-instantiate "$i")
for j in $storeExprs; do
echo "store expression is $(readlink "$j")" >&2
echo "store expression is $j" >&2
done
# Build the resulting store derivation.
prefix=
if test "$addOutLink" = 0; then prefix=.nix-build-tmp-; fi
outPaths=$(@bindir@/nix-store \
--add-root ./${prefix}result --indirect \
-rv $extraArgs $storeExprs)
outPaths=$(nix-store -qnfv $extraArgs $storeExprs)
for j in $outPaths; do
echo "$(readlink "$j")"
echo "$j"
if test -z "$noLink"; then
if test -L result; then
rm result
elif test -e result; then
echo "cannot remove \`result' (not a symlink)"
exit 1
fi
ln -s "$j" result
fi
done
;;
esac
done
Regular → Executable
+28 -70
View File
@@ -48,28 +48,11 @@ sub addChannel {
}
# Remove a channel from the file $channelsList;
sub removeChannel {
my $url = shift;
my @left = ();
readChannels;
foreach my $url2 (@channels) {
push @left, $url2 if $url ne $url2;
}
@channels = @left;
writeChannels;
}
# Fetch Nix expressions and pull cache manifests from the subscribed
# channels.
sub update {
readChannels;
# Get rid of all the old substitutes.
system "@bindir@/nix-store --clear-substitutes";
die "cannot clear substitutes" if ($? != 0);
# Pull cache manifests.
foreach my $url (@channels) {
print "pulling cache manifest from `$url'\n";
@@ -81,94 +64,69 @@ sub update {
# expressions.
my $nixExpr = "[";
foreach my $url (@channels) {
my $fullURL = "$url/nixexprs.tar.bz2";
$ENV{"PRINT_PATH"} = 1;
my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
die "cannot fetch `$fullURL'" if $? != 0;
chomp $path;
$nixExpr .= $path . " ";
my $hash = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`
or die "cannot fetch `$fullURL'";
chomp $hash;
# !!! escaping
$nixExpr .= "((import @datadir@/nix/corepkgs/fetchurl) " .
"{url = $fullURL; md5 = \"$hash\"; system = \"@system@\";}) "
}
$nixExpr .= "]";
$nixExpr =
"(import @datadir@/nix/corepkgs/channels/unpack.nix) " .
"{inputs = $nixExpr; system = \"@system@\";}";
# Figure out a name for the GC root.
my $userName = getpwuid($<);
die "who ARE you? go away" unless defined $userName;
my $rootFile = "$rootsDir/$userName";
# Instantiate the Nix expression.
my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate --add-root '$rootFile'.tmp -`
my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
or die "cannot instantiate Nix expression";
chomp $storeExpr;
# Build the resulting derivation.
my $outPath = `nix-store --add-root '$rootFile' -r '$storeExpr'`
# Register the store expression as a root of the garbage
# collector.
my $userName = getpwuid($<);
die "who ARE you? go away" unless defined $userName;
my $rootFile = "$rootsDir/$userName.gcroot";
my $tmpRootFile = "$rootsDir/$userName-tmp.gcroot";
open ROOT, ">$tmpRootFile" or die "cannot create `$tmpRootFile': $!";
print ROOT "$storeExpr";
close ROOT;
# Realise the store expression.
my $outPath = `nix-store -qnf '$storeExpr'`
or die "cannot realise store expression";
chomp $outPath;
unlink "$rootFile.tmp";
# Make it the default Nix expression for `nix-env'.
system "@bindir@/nix-env --import '$outPath'";
die "cannot pull set default Nix expression to `$outPath'" if ($? != 0);
rename $tmpRootFile, $rootFile or die "cannot rename `$tmpRootFile' to `$rootFile': $!";
}
sub usageError {
print STDERR <<EOF;
Usage:
nix-channel --add URL
nix-channel --remove URL
nix-channel --list
nix-channel --update
EOF
exit 1;
}
usageError if scalar @ARGV == 0;
while (scalar @ARGV) {
my $arg = shift @ARGV;
if ($arg eq "--add") {
usageError if scalar @ARGV != 1;
die "syntax: nix-channel --add URL" if (scalar @ARGV != 1);
addChannel (shift @ARGV);
last;
}
if ($arg eq "--remove") {
usageError if scalar @ARGV != 1;
removeChannel (shift @ARGV);
last;
}
if ($arg eq "--list") {
usageError if scalar @ARGV != 0;
readChannels;
foreach my $url (@channels) {
print "$url\n";
}
last;
}
elsif ($arg eq "--update") {
usageError if scalar @ARGV != 0;
die "syntax: nix-channel --update" if (scalar @ARGV != 0);
update;
last;
}
elsif ($arg eq "--help") {
usageError;
}
else {
die "unknown argument `$arg'; try `--help'";
die "unknown argument `$arg'";
}
}
Regular → Executable
+90 -2
View File
@@ -1,2 +1,90 @@
#! @shell@ -e
exec @bindir@/nix-store --gc "$@"
#! @perl@ -w
use strict;
use IPC::Open2;
my $rootsDir = "@localstatedir@/nix/gcroots";
my $storeDir = "@storedir@";
my %alive;
my $gcOper = "--delete";
my $minAge = 0;
my @roots = ();
# Parse the command line.
for (my $i = 0; $i < scalar @ARGV; $i++) {
my $arg = $ARGV[$i];
if ($arg eq "--delete" || $arg eq "--print-live" || $arg eq "--print-dead") {
$gcOper = $arg;
}
elsif ($arg eq "--min-age") {
$i++;
$minAge = undef;
$minAge = $ARGV[$i];
die "invalid minimum age" unless defined $minAge && $minAge =~ /^\d*$/;
}
else { die "unknown argument `$arg'" };
}
# Read all GC roots from the given file.
sub readRoots {
my $fileName = shift;
open ROOT, "<$fileName" or die "cannot open `$fileName': $!";
while (<ROOT>) {
chomp;
foreach my $root (split ' ') {
die "bad root `$root' in file `$fileName'"
unless $root =~ /^\S+$/;
push @roots, $root;
}
}
close ROOT;
}
# Recursively finds all *.gcroot files in the given directory.
sub findRoots;
sub findRoots {
my $followSymlinks = shift;
my $dir = shift;
opendir(DIR, $dir) or die "cannot open directory `$dir': $!";
my @names = readdir DIR or die "cannot read directory `$dir': $!";
closedir DIR;
foreach my $name (@names) {
next if $name eq "." || $name eq "..";
$name = $dir . "/" . $name;
if ($name =~ /.gcroot$/ && -f $name) {
readRoots $name;
}
elsif (-d $name) {
if ($followSymlinks || !-l $name) {
findRoots 0, $name;
}
}
}
}
# Find GC roots, starting at $rootsDir.
findRoots 1, $rootsDir;
# Run the collector with the roots we found.
my $pid = open2(">&1", \*WRITE, "@bindir@/nix-store --gc $gcOper --min-age $minAge")
or die "cannot run `nix-store --gc'";
foreach my $root (@roots) {
print WRITE "$root\n";
}
close WRITE;
waitpid $pid, 0;
$? == 0 or die "`nix-store --gc' failed";
+21 -35
View File
@@ -3,49 +3,35 @@
use strict;
use POSIX qw(tmpnam);
my $pkgFile = $ARGV[0];
die unless defined $pkgFile;
my $pkgfile = $ARGV[0];
die unless defined $pkgfile;
my $tmpdir;
do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
# Re-execute in a terminal, if necessary, so that if we're executed
# from a web browser, the user gets to see us.
if (!defined $ENV{"NIX_HAVE_TERMINAL"}) {
$ENV{"NIX_HAVE_TERMINAL"} = "1";
$ENV{"LD_LIBRARY_PATH"} = "";
exec("xterm", "-e", "@shell@", "-c", "@bindir@/nix-install-package '$pkgFile' || read");
die "cannot execute `xterm'";
}
# !!! remove tmpdir on exit
print "Unpacking $pkgfile in $tmpdir...\n";
system "bunzip2 < $pkgfile | (cd $tmpdir && tar xf -)";
die if $?;
# Read and parse the package file.
open PKGFILE, "<$pkgFile" or die "cannot open `$pkgFile': $!";
my $contents = <PKGFILE>;
close PKGFILE;
print "This package contains the following derivations:\n";
system "@bindir@/nix-env -qasf $tmpdir/default.nix";
die if $?;
$contents =~ /^\s*(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)/ or die "invalid package contents";
my $version = $1;
my $manifestURL = $2;
my $drvName = $3;
my $system = $4;
my $drvPath = $5;
my $outPath = $6;
die "invalid package version `$version'" unless $version eq "NIXPKG1";
# Ask confirmation.
print "Do you want to install `$drvName' (Y/N)? ";
print "Do you wish to install these (Y/N)? ";
my $reply = <STDIN>;
chomp $reply;
exit if $reply ne "y" && $reply ne "Y";
exit if (!($reply eq "y"));
print "\nPulling manifests...\n";
system "@bindir@/nix-pull '$manifestURL'";
die if $? != 0;
print "Pulling caches...\n";
system "@bindir@/nix-pull `cat $tmpdir/caches`";
die if $?;
print "\nInstalling package...\n";
system "@bindir@/nix-env -i '$outPath'";
die if $? != 0;
print "Installing package...\n";
system "@bindir@/nix-env -if $tmpdir/default.nix '*'";
die if $?;
print "\nInstallation succeeded! Press Enter to continue.\n";
print "Installation succeeded! Press Enter to continue.\n";
<STDIN>;
+30 -47
View File
@@ -1,66 +1,49 @@
#! @shell@ -e
url=$1
expHash=$2
hashType=$NIX_HASH_ALGO
if test -z "$hashType"; then
hashType=md5
fi
hashFormat=
if test "$hashType" != "md5"; then
hashFormat=--base32
fi
if test -z "$url"; then
echo "syntax: nix-prefetch-url URL [EXPECTED-HASH]" >&2
echo "syntax: nix-prefetch-url URL" >&2
exit 1
fi
name=$(basename "$url")
if test -z "$name"; then echo "invalid url"; exit 1; fi
# !!! race
tmpPath1=@storedir@/nix-prefetch-url-$$
# If the hash was given, a file with that hash may already be in the
# store.
if test -n "$expHash"; then
finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$expHash" "$name")
if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
finalPath=
fi
hash=$expHash
# Test whether we have write permission in the store. If not, fetch
# to /tmp and don't copy to the store. This is a hack to make this
# script at least work somewhat in setuid installations.
if ! touch $tmpPath1 2> /dev/null; then
echo "(cannot write to the store, result won't be cached)" >&2
dummyMode=1
tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
fi
# Perform the checkout.
@curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
# If we don't know the hash or a file with that hash doesn't exist,
# download the file and add it to the store.
if test -z "$finalPath"; then
# Compute the hash.
hash=$(@bindir@/nix-hash --flat $tmpPath1)
echo "hash is $hash" >&2
tmpPath=/tmp/nix-prefetch-url-$$ # !!! security?
tmpFile=$tmpPath/$name
mkdir $tmpPath
# Perform the download.
@curl@ --fail --location --max-redirs 20 "$url" > $tmpFile
# Compute the hash.
hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
# Add the downloaded file to the Nix store.
finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
if test -n "$expHash" -a "$expHash" != "$hash"; then
echo "hash mismatch for URL \`$url'"
exit 1
fi
# Rename it so that the fetchurl builder can find it.
if test "$dummyMode" != 1; then
tmpPath2=@storedir@/nix-prefetch-url-$hash
test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
fi
# Create a Nix expression that does a fetchurl.
storeExpr=$( \
echo "(import @datadir@/nix/corepkgs/fetchurl) \
{url = $url; md5 = \"$hash\"; system = \"@system@\";}" \
| @bindir@/nix-instantiate -)
if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
# Realise it.
finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
echo "path is $finalPath" >&2
rm -rf $tmpPath1 $tmpPath2 || true
echo $hash
+55 -63
View File
@@ -10,92 +10,84 @@ do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
my $manifest = "$tmpdir/manifest";
my $confFile = "@sysconfdir@/nix/prebuilts.conf";
END { unlink $manifest; rmdir $tmpdir; }
my $binDir = $ENV{"NIX_BIN_DIR"};
$binDir = "@bindir@" unless defined $binDir;
my $libexecDir = $ENV{"NIX_LIBEXEC_DIR"};
$libexecDir = "@libexecdir@" unless defined $libexecDir;
my $stateDir = $ENV{"NIX_STATE_DIR"};
$stateDir = "@localstatedir@/nix" unless defined $stateDir;
# Prevent access problems in shared-stored installations.
umask 0022;
#END { unlink $manifest; rmdir $tmpdir; }
# Obtain URLs either from the command line or from a configuration file.
my %narFiles;
my %patches;
my %storePaths2urls;
my %urls2hashes;
my %successors;
sub processURL {
sub doURL {
my $url = shift;
$url =~ s/\/$//;
print "obtaining list of Nix archives at $url...\n";
system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
"'$url' > '$manifest'") == 0
or die "curl failed: $?";
if (readManifest($manifest, \%narFiles, \%patches, \%successors) < 3) {
die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
}
my $baseName = "unnamed";
if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
$baseName = $1;
}
my $hash = `$binDir/nix-hash --flat '$manifest'`
or die "cannot hash `$manifest'";
chomp $hash;
my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
system("mv -f '$manifest' '$finalPath'") == 0
or die "cannot move `$manifest' to `$finalPath";
processURL $manifest, $url, \%storePaths2urls, \%urls2hashes, \%successors;
}
while (@ARGV) {
my $url = shift @ARGV;
processURL $url;
if (scalar @ARGV > 0) {
while (@ARGV) {
my $url = shift @ARGV;
doURL $url;
}
} else {
open CONFFILE, "<$confFile";
while (<CONFFILE>) {
chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1;
doURL $url;
}
}
close CONFFILE;
}
my $size = scalar (keys %narFiles);
my $size = scalar (keys %storePaths2urls);
print "$size store paths in manifest\n";
# Instantiate a store expression that builds the substitute program
# (the program that fetches URLs and unpacks them into the store).
my $nixExpr =
"(import @datadir@/nix/corepkgs/nix-pull) " .
"{system = \"@system@\";}";
print STDERR "instantiating store expression...\n";
my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
or die "cannot instantiate Nix expression";
chomp $storeExpr;
# Register all substitutes.
print STDERR "registering substitutes...\n";
my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --register-substitutes")
my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
or die "cannot run nix-store";
close READ;
foreach my $storePath (keys %narFiles) {
my $narFileList = $narFiles{$storePath};
foreach my $narFile (@{$narFileList}) {
print WRITE "$storePath\n";
print WRITE "$narFile->{deriver}\n";
print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
print WRITE "0\n";
my @references = split " ", $narFile->{references};
my $count = scalar @references;
print WRITE "$count\n";
foreach my $reference (@references) {
print WRITE "$reference\n";
}
}
foreach my $storePath (keys %storePaths2urls) {
print WRITE "$storePath\n";
print WRITE "$storeExpr\n";
print WRITE "/fetch\n";
print WRITE "2\n";
print WRITE "$storePaths2urls{$storePath}\n";
print WRITE "$urls2hashes{$storePaths2urls{$storePath}}\n";
}
close WRITE;
waitpid $pid, 0;
$? == 0 or die "nix-store failed";
# Register all successors.
print STDERR "registering successors...\n";
my @sucs = %successors;
while (scalar @sucs > 0) {
my $n = scalar @sucs;
if ($n > 256) { $n = 256 };
my @sucs2 = @sucs[0..$n - 1];
@sucs = @sucs[$n..scalar @sucs - 1];
system "@bindir@/nix-store --successor @sucs2";
if ($?) { die "`nix-store --successor' failed"; }
}
+81 -160
View File
@@ -1,11 +1,7 @@
#! @perl@ -w -I@libexecdir@/nix
#! @perl@ -w
use strict;
use IPC::Open2;
use POSIX qw(tmpnam);
use readmanifest;
my $hashAlgo = "sha256";
my $tmpdir;
do { $tmpdir = tmpnam(); }
@@ -20,65 +16,35 @@ my $curl = "@curl@ --fail --silent";
my $extraCurlFlags = ${ENV{'CURL_FLAGS'}};
$curl = "$curl $extraCurlFlags" if defined $extraCurlFlags;
my $binDir = $ENV{"NIX_BIN_DIR"};
$binDir = "@bindir@" unless defined $binDir;
my $dataDir = $ENV{"NIX_DATA_DIR"};
$dataDir = "@datadir@" unless defined $dataDir;
# Parse the command line.
my $localCopy;
my $localArchivesDir;
my $localManifestFile;
my $archivesPutURL;
my $archivesGetURL;
my $manifestPutURL;
if ($ARGV[0] eq "--copy") {
die "syntax: nix-push --copy ARCHIVES_DIR MANIFEST_FILE PATHS...\n" if scalar @ARGV < 3;
$localCopy = 1;
shift @ARGV;
$localArchivesDir = shift @ARGV;
$localManifestFile = shift @ARGV;
}
else {
die "syntax: nix-push ARCHIVES_PUT_URL ARCHIVES_GET_URL " .
"MANIFEST_PUT_URL PATHS...\n" if scalar @ARGV < 3;
$localCopy = 0;
$archivesPutURL = shift @ARGV;
$archivesGetURL = shift @ARGV;
$manifestPutURL = shift @ARGV;
}
my $archives_put_url = shift @ARGV;
my $archives_get_url = shift @ARGV;
my $manifest_put_url = shift @ARGV;
# From the given store paths, determine the set of requisite store
# paths, i.e, the paths required to realise them.
my %storePaths;
# From the given store expressions, determine the requisite store
# paths.
my %storepaths;
foreach my $path (@ARGV) {
die unless $path =~ /^\//;
foreach my $storeexpr (@ARGV) {
die unless $storeexpr =~ /^\//;
# Get all paths referenced by the normalisation of the given
# Nix expression.
my $pid = open2(\*READ, \*WRITE,
"$binDir/nix-store --query --requisites --force-realise " .
"--include-outputs '$path'") or die;
close WRITE;
while (<READ>) {
system "@bindir@/nix-store --realise $storeexpr > /dev/null";
die if ($?);
open PATHS, "@bindir@/nix-store --query --requisites --include-successors $storeexpr 2> /dev/null |" or die;
while (<PATHS>) {
chomp;
die "bad: $_" unless /^\//;
$storePaths{$_} = "";
$storepaths{$_} = "";
}
close READ;
waitpid $pid, 0;
$? == 0 or die "nix-store failed";
close PATHS;
}
my @storePaths = keys %storePaths;
my @storepaths = keys %storepaths;
# For each path, create a Nix expression that turns the path into
@@ -86,13 +52,14 @@ my @storePaths = keys %storePaths;
open NIX, ">$nixfile";
print NIX "[";
foreach my $storePath (@storePaths) {
die unless ($storePath =~ /\/[0-9a-z]{32}.*$/);
foreach my $storepath (@storepaths) {
die unless ($storepath =~ /\/[0-9a-z]{32}.*$/);
# Construct a Nix expression that creates a Nix archive.
my $nixexpr =
"((import $dataDir/nix/corepkgs/nar/nar.nix) " .
"{path = \"$storePath\"; system = \"@system@\"; hashAlgo = \"$hashAlgo\";}) ";
"((import @datadir@/nix/corepkgs/nar/nar.nix) " .
# !!! $storepath should be represented as a closure
"{path = \"$storepath\"; system = \"@system@\";}) ";
print NIX $nixexpr;
}
@@ -102,158 +69,112 @@ close NIX;
# Instantiate store expressions from the Nix expression.
my @storeExprs;
my @storeexprs;
print STDERR "instantiating store expressions...\n";
my $pid = open2(\*READ, \*WRITE, "$binDir/nix-instantiate $nixfile")
or die "cannot run nix-instantiate";
close WRITE;
while (<READ>) {
open STOREEXPRS, "@bindir@/nix-instantiate $nixfile |" or die "cannot run nix-instantiate";
while (<STOREEXPRS>) {
chomp;
die unless /^\//;
push @storeExprs, $_;
push @storeexprs, $_;
}
close READ;
waitpid $pid, 0;
$? == 0 or die "nix-instantiate failed";
close STOREEXPRS;
# Realise the store expressions.
print STDERR "creating archives...\n";
my @narPaths;
my @narpaths;
my @tmp = @storeExprs;
my @tmp = @storeexprs;
while (scalar @tmp > 0) {
my $n = scalar @tmp;
if ($n > 256) { $n = 256 };
my @tmp2 = @tmp[0..$n - 1];
@tmp = @tmp[$n..scalar @tmp - 1];
my $pid = open2(\*READ, \*WRITE, "$binDir/nix-store --realise @tmp2")
or die "cannot run nix-store";
close WRITE;
while (<READ>) {
system "@bindir@/nix-store --realise @tmp2 > /dev/null";
if ($?) { die "`nix-store --realise' failed"; }
open NARPATHS, "@bindir@/nix-store --query --list @tmp2 |" or die "cannot run nix";
while (<NARPATHS>) {
chomp;
die unless (/^\//);
push @narPaths, "$_";
push @narpaths, "$_";
}
close READ;
waitpid $pid, 0;
$? == 0 or die "nix-store failed";
close NARPATHS;
}
# Create the manifest.
print STDERR "creating manifest...\n";
my %narFiles;
my %patches;
open MANIFEST, ">$manifest";
my @narArchives;
for (my $n = 0; $n < scalar @storePaths; $n++) {
my $storePath = $storePaths[$n];
my $narDir = $narPaths[$n];
my @nararchives;
for (my $n = 0; $n < scalar @storepaths; $n++) {
my $storepath = $storepaths[$n];
my $nardir = $narpaths[$n];
$storePath =~ /\/([^\/]*)$/;
$storepath =~ /\/([^\/]*)$/;
my $basename = $1;
defined $basename or die;
open HASH, "$narDir/narbz2-hash" or die "cannot open narbz2-hash";
my $narbz2Hash = <HASH>;
chomp $narbz2Hash;
$narbz2Hash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narname = "$basename.nar.bz2";
open HASH, "$narDir/nar-hash" or die "cannot open nar-hash";
my $narHash = <HASH>;
chomp $narHash;
$narHash =~ /^[0-9a-z]+$/ or die "invalid hash";
close HASH;
my $narName = "$narbz2Hash.nar.bz2";
my $narfile = "$nardir/$narname";
(-f $narfile) or die "narfile for $storepath not found";
push @nararchives, $narfile;
my $narFile = "$narDir/$narName";
(-f $narFile) or die "narfile for $storePath not found";
push @narArchives, $narFile;
open MD5, "$nardir/md5" or die "cannot open hash";
my $hash = <MD5>;
chomp $hash;
$hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
close MD5;
my $narbz2Size = (stat $narFile)[7];
print MANIFEST "{\n";
print MANIFEST " StorePath: $storepath\n";
print MANIFEST " NarURL: $archives_get_url/$narname\n";
print MANIFEST " MD5: $hash\n";
my $references = `$binDir/nix-store --query --references '$storePath'`;
die "cannot query references for `$storePath'" if $? != 0;
$references = join(" ", split(" ", $references));
my $deriver = `$binDir/nix-store --query --deriver '$storePath'`;
die "cannot query deriver for `$storePath'" if $? != 0;
chomp $deriver;
$deriver = "" if $deriver eq "unknown-deriver";
my $url;
if ($localCopy) {
$url = "file://$localArchivesDir/$narName";
} else {
$url = "$archivesGetURL/$narName";
}
$narFiles{$storePath} = [
{ url => $url
, hash => "$hashAlgo:$narbz2Hash"
, size => $narbz2Size
, narHash => "$hashAlgo:$narHash"
, references => $references
, deriver => $deriver
if ($storepath =~ /\.store$/) {
open PREDS, "@bindir@/nix-store --query --predecessors $storepath |" or die "cannot run nix";
while (<PREDS>) {
chomp;
die unless (/^\//);
my $pred = $_;
# Only include predecessors that are themselves being
# pushed.
if (defined $storepaths{$pred}) {
print MANIFEST " SuccOf: $pred\n";
}
}
];
close PREDS;
}
print MANIFEST "}\n";
}
writeManifest $manifest, \%narFiles, \%patches;
sub copyFile {
my $src = shift;
my $dst = shift;
system("cp '$src' '$dst.tmp'") == 0 or die "cannot copy file";
rename("$dst.tmp", "$dst") or die "cannot rename file";
}
close MANIFEST;
# Upload the archives.
print STDERR "uploading archives...\n";
foreach my $nararchive (@nararchives) {
sub archiveExists {
my $name = shift;
print STDERR " HEAD on $archivesGetURL/$name\n";
return system("$curl --head $archivesGetURL/$name > /dev/null") == 0;
}
foreach my $narArchive (@narArchives) {
$narArchive =~ /\/([^\/]*)$/;
$nararchive =~ /\/([^\/]*)$/;
my $basename = $1;
if ($localCopy) {
if (! -f "$localArchivesDir/$basename") {
print STDERR " $narArchive\n";
copyFile $narArchive, "$localArchivesDir/$basename";
}
}
else {
if (!archiveExists("$basename")) {
print STDERR " $narArchive\n";
system("$curl --show-error --upload-file " .
"'$narArchive' '$archivesPutURL/$basename' > /dev/null") == 0 or
die "curl failed on $narArchive: $?";
}
if (system("$curl --head $archives_get_url/$basename > /dev/null") != 0) {
print STDERR " $nararchive\n";
system("$curl --show-error --upload-file " .
"'$nararchive' '$archives_put_url/$basename' > /dev/null") == 0 or
die "curl failed on $nararchive: $?";
}
}
# Upload the manifest.
print STDERR "uploading manifest...\n";
if ($localCopy) {
copyFile $manifest, $localManifestFile;
} else {
system("$curl --show-error --upload-file " .
"'$manifest' '$manifestPutURL' > /dev/null") == 0 or
die "curl failed on $manifest: $?";
}
system("$curl --show-error --upload-file " .
"'$manifest' '$manifest_put_url' > /dev/null") == 0 or
die "curl failed on $manifest: $?";
-69
View File
@@ -1,69 +0,0 @@
#! /usr/bin/perl -w
use strict;
{ my $ofh = select STDOUT;
$| = 1;
select $ofh;
}
#my @paths = ("/nix/store/caef3a49150506d233f474322a824e50-glibc-2.3.3", "/nix/store/a8a9d585d1ad4b1bc911be7743b3b996-glibc-2.3.3");
my @paths = ("/nix/store");
my $tmpfile = "/tmp/nix-optimise-hash-list";
#my $tmpfile = "/data/nix-optimise-hash-list";
system("find @paths -type f -print0 | xargs -0 md5sum -- > $tmpfile") == 0
or die "cannot hash store files";
system("sort $tmpfile > $tmpfile.sorted") == 0
or die "cannot sort list";
open LIST, "<$tmpfile.sorted" or die;
my $prevFile;
my $prevHash;
my $totalSpace = 0;
my $savedSpace = 0;
my $files = 0;
while (<LIST>) {
# print "D";
/^([0-9a-f]*)\s+(.*)$/ or die;
my $curFile = $2;
my $curHash = $1;
# print "A";
my $fileSize = (stat $curFile)[7];
# print "B";
# my $fileSize = 1;
$totalSpace += $fileSize;
if (defined $prevHash && $curHash eq $prevHash) {
# print "$curFile = $prevFile\n";
$savedSpace += $fileSize;
} else {
$prevFile = $curFile;
$prevHash = $curHash;
}
print "." if ($files++ % 100 == 0);
#print ".";
# print "C";
}
print "\n";
print "total space = $totalSpace\n";
print "saved space = $savedSpace\n";
my $savings = ($savedSpace / $totalSpace) * 100.0;
print "savings = $savings %\n";
close LIST;
+33 -177
View File
@@ -1,64 +1,27 @@
use strict;
sub addPatch {
my $patches = shift;
my $storePath = shift;
my $patch = shift;
my $allowConflicts = shift;
$$patches{$storePath} = []
unless defined $$patches{$storePath};
my $patchList = $$patches{$storePath};
my $found = 0;
foreach my $patch2 (@{$patchList}) {
if ($patch2->{url} eq $patch->{url}) {
if ($patch2->{hash} eq $patch->{hash}) {
$found = 1 if ($patch2->{basePath} eq $patch->{basePath});
} else {
die "conflicting hashes for URL $patch->{url}, " .
"namely $patch2->{hash} and $patch->{hash}"
unless $allowConflicts;
}
}
}
push @{$patchList}, $patch if !$found;
return !$found;
}
sub readManifest {
sub processURL {
my $manifest = shift;
my $narFiles = shift;
my $patches = shift;
my $url = shift;
my $storePaths2urls = shift;
my $urls2hashes = shift;
my $successors = shift;
my $allowConflicts = shift;
$allowConflicts = 0 unless defined $allowConflicts;
open MANIFEST, "<$manifest"
or die "cannot open `$manifest': $!";
$url =~ s/\/$//;
print "obtaining list of Nix archives at $url...\n";
system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
"'$url' > '$manifest'") == 0
or die "curl failed: $?";
open MANIFEST, "<$manifest";
my $inside = 0;
my $type;
my $manifestVersion = 2;
my $storePath;
my $url;
my $narurl;
my $hash;
my $size;
my @preds;
my $basePath;
my $baseHash;
my $patchType;
my $narHash;
my $references;
my $deriver;
my $hashAlgo;
while (<MANIFEST>) {
chomp;
@@ -66,150 +29,43 @@ sub readManifest {
next if (/^$/);
if (!$inside) {
if (/^\s*(\w*)\s*\{$/) {
$type = $1;
$type = "narfile" if $type eq "";
if (/^\{$/) {
$inside = 1;
undef $storePath;
undef $url;
undef $narurl;
undef $hash;
undef $size;
@preds = ();
undef $narHash;
undef $basePath;
undef $baseHash;
undef $patchType;
$references = "";
$deriver = "";
$hashAlgo = "md5";
}
else { die "bad line: $_"; }
} else {
if (/^\}$/) {
$inside = 0;
if ($type eq "narfile") {
$$storePaths2urls{$storePath} = $narurl;
$$urls2hashes{$narurl} = $hash;
$$narFiles{$storePath} = []
unless defined $$narFiles{$storePath};
my $narFileList = $$narFiles{$storePath};
my $found = 0;
foreach my $narFile (@{$narFileList}) {
if ($narFile->{url} eq $url) {
if ($narFile->{hash} eq $hash) {
$found = 1;
} else {
die "conflicting hashes for URL $url, " .
"namely $narFile->{hash} and $hash"
unless $allowConflicts;
}
}
}
if (!$found) {
push @{$narFileList},
{ url => $url, hash => $hash, size => $size
, narHash => $narHash, references => $references
, deriver => $deriver, hashAlgo => $hashAlgo
};
}
foreach my $p (@preds) {
$$successors{$p} = $storePath;
}
}
elsif ($type eq "patch") {
addPatch $patches, $storePath,
{ url => $url, hash => $hash, size => $size
, basePath => $basePath, baseHash => $baseHash
, narHash => $narHash, patchType => $patchType
, hashAlgo => $hashAlgo
}, $allowConflicts;
foreach my $p (@preds) {
$$successors{$p} = $storePath;
}
}
elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; }
elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { push @preds, $1; }
elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; }
elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; }
elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; }
# Compatibility;
elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; }
elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
$storePath = $1;
}
elsif (/^\s*NarURL:\s*(\S+)\s*$/) {
$narurl = $1;
}
elsif (/^\s*MD5:\s*(\S+)\s*$/) {
$hash = $1;
}
elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
push @preds, $1;
}
else { die "bad line: $_"; }
}
}
close MANIFEST;
return $manifestVersion;
}
sub writeManifest
{
my $manifest = shift;
my $narFiles = shift;
my $patches = shift;
open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
print MANIFEST "version {\n";
print MANIFEST " ManifestVersion: 3\n";
print MANIFEST "}\n";
foreach my $storePath (keys %{$narFiles}) {
my $narFileList = $$narFiles{$storePath};
foreach my $narFile (@{$narFileList}) {
print MANIFEST "{\n";
print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $narFile->{url}\n";
print MANIFEST " Hash: $narFile->{hash}\n";
print MANIFEST " NarHash: $narFile->{narHash}\n";
print MANIFEST " Size: $narFile->{size}\n";
print MANIFEST " References: $narFile->{references}\n"
if defined $narFile->{references} && $narFile->{references} ne "";
print MANIFEST " Deriver: $narFile->{deriver}\n"
if defined $narFile->{deriver} && $narFile->{deriver} ne "";
print MANIFEST "}\n";
}
}
foreach my $storePath (keys %{$patches}) {
my $patchList = $$patches{$storePath};
foreach my $patch (@{$patchList}) {
print MANIFEST "patch {\n";
print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $patch->{url}\n";
print MANIFEST " Hash: $patch->{hash}\n";
print MANIFEST " NarHash: $patch->{narHash}\n";
print MANIFEST " Size: $patch->{size}\n";
print MANIFEST " BasePath: $patch->{basePath}\n";
print MANIFEST " BaseHash: $patch->{baseHash}\n";
print MANIFEST " Type: $patch->{patchType}\n";
print MANIFEST "}\n";
}
}
close MANIFEST;
rename("$manifest.tmp", $manifest)
or die "cannot rename $manifest.tmp: $!";
}
-19
View File
@@ -1,19 +0,0 @@
#! /usr/bin/perl -w -I/home/eelco/nix/scripts
use strict;
use readmanifest;
for my $p (@ARGV) {
my %narFiles;
my %patches;
my %successors;
readManifest $p,
\%narFiles, \%patches, \%successors;
%patches = ();
writeManifest $p,
\%narFiles, \%patches, \%successors;
}
-53
View File
@@ -1,53 +0,0 @@
#! /usr/bin/perl -w -I.
use strict;
use readmanifest;
die unless scalar @ARGV == 2;
my $cache = $ARGV[0];
my $manifest = $ARGV[1];
my %narFiles;
my %patches;
my %successors;
readManifest $manifest, \%narFiles, \%patches, \%successors;
foreach my $storePath (keys %narFiles) {
my $narFileList = $narFiles{$storePath};
foreach my $narFile (@{$narFileList}) {
if (!defined $narFile->{size} or
!defined $narFile->{narHash})
{
$narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1;
my $fn = "$cache/$1";
my @info = stat $fn or die;
$narFile->{size} = $info[7];
my $narHash;
my $hashFile = "$fn.NARHASH";
if (-e $hashFile) {
open HASH, "<$hashFile" or die;
$narHash = <HASH>;
close HASH;
} else {
print "$fn\n";
$narHash = `bunzip2 < '$fn' | nix-hash --flat /dev/stdin` or die;
open HASH, ">$hashFile" or die;
print HASH $narHash;
close HASH;
}
chomp $narHash;
$narFile->{narHash} = $narHash;
}
}
}
if (! -e "$manifest.backup") {
system "mv --reply=no '$manifest' '$manifest.backup'";
}
writeManifest $manifest, \%narFiles, \%patches, \%successors;
+1 -1
View File
@@ -1,5 +1,5 @@
SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
libexpr nix-instantiate nix-env log2xml bsdiff-4.2
libexpr nix-instantiate nix-env log2xml
EXTRA_DIST = aterm-helper.pl
+2 -12
View File
@@ -56,7 +56,6 @@ while (<STDIN>) {
my $unpack = "";
my $n = 1;
foreach my $type (@types) {
my $realType = $type;
$args .= ", ";
if ($type eq "string") {
# $args .= "(ATerm) ATmakeAppl0(ATmakeAFun((char *) e$n, 0, ATtrue))";
@@ -84,9 +83,6 @@ while (<STDIN>) {
$unpack .= " e$n = (ATermList) ATgetArgument(e, $m);\n";
} elsif ($type eq "ATermBlob") {
$unpack .= " e$n = (ATermBlob) ATgetArgument(e, $m);\n";
} elsif ($realType eq "string") {
$unpack .= " e$n = ATgetArgument(e, $m);\n";
$unpack .= " if (ATgetType(e$n) != AT_APPL) return false;\n";
} else {
$unpack .= " e$n = ATgetArgument(e, $m);\n";
}
@@ -100,18 +96,12 @@ while (<STDIN>) {
print IMPL "AFun sym$funname = 0;\n";
print HEADER "static inline $result make$funname($formals) {\n";
if ($arity <= 6) {
print HEADER " return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
} else {
$args =~ s/^,//;
print HEADER " ATerm array[$arity] = {$args};\n";
print HEADER " return (ATerm) ATmakeApplArray(sym$funname, array);\n";
}
print HEADER " return (ATerm) ATmakeAppl$arity(sym$funname$args);\n";
print HEADER "}\n\n";
print HEADER "#ifdef __cplusplus\n";
print HEADER "static inline bool match$funname(ATerm e$formals2) {\n";
print HEADER " if (ATgetType(e) != AT_APPL || (AFun) ATgetAFun(e) != sym$funname) return false;\n";
print HEADER " if (ATgetType(e) != AT_APPL || ATgetAFun(e) != sym$funname) return false;\n";
print HEADER "$unpack";
print HEADER " return true;\n";
print HEADER "}\n";
-121
View File
@@ -1,121 +0,0 @@
BSD Protection License
February 2002
Preamble
--------
The Berkeley Software Distribution ("BSD") license has proven very effective
over the years at allowing for a wide spread of work throughout both
commercial and non-commercial products. For programmers whose primary
intention is to improve the general quality of available software, it is
arguable that there is no better license than the BSD license, as it
permits improvements to be used wherever they will help, without idealogical
or metallic constraint.
This is of particular value to those who produce reference implementations
of proposed standards: The case of TCP/IP clearly illustrates that freely
and universally available implementations leads the rapid acceptance of
standards -- often even being used instead of a de jure standard (eg, OSI
network models).
With the rapid proliferation of software licensed under the GNU General
Public License, however, the continued success of this role is called into
question. Given that the inclusion of a few lines of "GPL-tainted" work
into a larger body of work will result in restricted distribution -- and
given that further work will likely build upon the "tainted" portions,
making them difficult to remove at a future date -- there are inevitable
circumstances where authors would, in order to protect their goal of
providing for the widespread usage of their work, wish to guard against
such "GPL-taint".
In addition, one can imagine that companies which operate by producing and
selling (possibly closed-source) code would wish to protect themselves
against the rise of a GPL-licensed competitor. While under existing
licenses this would mean not releasing their code under any form of open
license, if a license existed under which they could incorporate any
improvements back into their own (commercial) products then they might be
far more willing to provide for non-closed distribution.
For the above reasons, we put forth this "BSD Protection License": A
license designed to retain the freedom granted by the BSD license to use
licensed works in a wide variety of settings, both non-commercial and
commercial, while protecting the work from having future contributors
restrict that freedom.
The precise terms and conditions for copying, distribution, and
modification follow.
BSD PROTECTION LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION, AND MODIFICATION
----------------------------------------------------------------
0. Definitions.
a) "Program", below, refers to any program or work distributed under
the terms of this license.
b) A "work based on the Program", below, refers to either the Program
or any derivative work under copyright law.
c) "Modification", below, refers to the act of creating derivative works.
d) "You", below, refers to each licensee.
1. Scope.
This license governs the copying, distribution, and modification of the
Program. Other activities are outside the scope of this license; The
act of running the Program is not restricted, and the output from the
Program is covered only if its contents constitute a work based on the
Program.
2. Verbatim copies.
You may copy and distribute verbatim copies of the Program as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice; keep
intact all the notices that refer to this License and to the absence of
any warranty; and give any other recipients of the Program a copy of this
License along with the Program.
3. Modification and redistribution under closed license.
You may modify your copy or copies of the Program, and distribute
the resulting derivative works, provided that you meet the
following conditions:
a) The copyright notice and disclaimer on the Program must be reproduced
and included in the source code, documentation, and/or other materials
provided in a manner in which such notices are normally distributed.
b) The derivative work must be clearly identified as such, in order that
it may not be confused with the original work.
c) The license under which the derivative work is distributed must
expressly prohibit the distribution of further derivative works.
4. Modification and redistribution under open license.
You may modify your copy or copies of the Program, and distribute
the resulting derivative works, provided that you meet the
following conditions:
a) The copyright notice and disclaimer on the Program must be reproduced
and included in the source code, documentation, and/or other materials
provided in a manner in which such notices are normally distributed.
b) You must clearly indicate the nature and date of any changes made
to the Program. The full details need not necessarily be included in
the individual modified files, provided that each modified file is
clearly marked as such and instructions are included on where the
full details of the modifications may be found.
c) You must cause any work that you distribute or publish, that in whole
or in part contains or is derived from the Program or any part
thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
5. Implied acceptance.
You may not copy or distribute the Program or any derivative works except
as expressly provided under this license. Consequently, any such action
will be taken as implied acceptance of the terms of this license.
6. NO WARRANTY.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
THE COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
TORT, EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
-7
View File
@@ -1,7 +0,0 @@
libexec_PROGRAMS = bsdiff bspatch
bsdiff_SOURCES = bsdiff.c
bspatch_SOURCES = bspatch.c
AM_CFLAGS = -O3 -DBZIP2=\"$(bzip2)\"
-18
View File
@@ -1,18 +0,0 @@
CFLAGS += -O3
.ifdef BZIP2
CFLAGS += -DBZIP2=\"${BZIP2}\"
.endif
PREFIX ?= /usr/local
INSTALL_PROGRAM ?= ${INSTALL} -c -s -m 555
INSTALL_MAN ?= ${INSTALL} -c -m 444
all: bsdiff bspatch
bsdiff: bsdiff.c
bspatch: bspatch.c
install:
${INSTALL_PROGRAM} bsdiff bspatch ${PREFIX}/bin
.ifndef WITHOUT_MAN
${INSTALL_MAN} bsdiff.1 bspatch.1 ${PREFIX}/man/man1
.endif
-36
View File
@@ -1,36 +0,0 @@
.Dd May 18, 2003
.Dt BSDIFF 1
.Os FreeBSD
.Sh NAME
.Nm bsdiff
.Nd generate a patch between two binary files
.Sh SYNOPSIS
.Nm
.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
.Sh DESCRIPTION
.Nm
compares
.Ao Ar oldfile Ac
to
.Ao Ar newfile Ac
and writes to
.Ao Ar patchfile Ac
a binary patch suitable for use by bspatch(1).
When
.Ao Ar oldfile Ac
and
.Ao Ar newfile Ac
are two versions of an executable program, the
patches produced are on average a factor of five smaller
than those produced by any other binary patch tool known
to the author.
.Pp
.Nm
uses memory equal to 17 times the size of
.Ao Ar oldfile Ac ,
and requires
an absolute minimum working set size of 8 times the size of oldfile.
.Sh SEE ALSO
.Xr bspatch 1
.Sh AUTHORS
.An Colin Percival Aq cperciva@daemonology.net
-385
View File
@@ -1,385 +0,0 @@
/*
bsdiff.c -- Binary patch generator.
Copyright 2003 Colin Percival
For the terms under which this work may be distributed, please see
the adjoining file "LICENSE".
*/
#ifndef BZIP2
#define BZIP2 "/usr/bin/bzip2"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <err.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/wait.h>
#define MIN(x,y) (((x)<(y)) ? (x) : (y))
void split(off_t *I,off_t *V,off_t start,off_t len,off_t h)
{
off_t i,j,k,x,tmp,jj,kk;
if(len<16) {
for(k=start;k<start+len;k+=j) {
j=1;x=V[I[k]+h];
for(i=1;k+i<start+len;i++) {
if(V[I[k+i]+h]<x) {
x=V[I[k+i]+h];
j=0;
};
if(V[I[k+i]+h]==x) {
tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp;
j++;
};
};
for(i=0;i<j;i++) V[I[k+i]]=k+j-1;
if(j==1) I[k]=-1;
};
return;
};
x=V[I[start+len/2]+h];
jj=0;kk=0;
for(i=start;i<start+len;i++) {
if(V[I[i]+h]<x) jj++;
if(V[I[i]+h]==x) kk++;
};
jj+=start;kk+=jj;
i=start;j=0;k=0;
while(i<jj) {
if(V[I[i]+h]<x) {
i++;
} else if(V[I[i]+h]==x) {
tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp;
j++;
} else {
tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp;
k++;
};
};
while(jj+j<kk) {
if(V[I[jj+j]+h]==x) {
j++;
} else {
tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp;
k++;
};
};
if(jj>start) split(I,V,start,jj-start,h);
for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1;
if(jj==kk-1) I[jj]=-1;
if(start+len>kk) split(I,V,kk,start+len-kk,h);
}
void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize)
{
off_t buckets[256];
off_t i,h,len;
for(i=0;i<256;i++) buckets[i]=0;
for(i=0;i<oldsize;i++) buckets[old[i]]++;
for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
for(i=255;i>0;i--) buckets[i]=buckets[i-1];
buckets[0]=0;
for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i;
I[0]=oldsize;
for(i=0;i<oldsize;i++) V[i]=buckets[old[i]];
V[oldsize]=0;
for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
I[0]=-1;
for(h=1;I[0]!=-(oldsize+1);h+=h) {
len=0;
for(i=0;i<oldsize+1;) {
if(I[i]<0) {
len-=I[i];
i-=I[i];
} else {
if(len) I[i-len]=-len;
len=V[I[i]]+1-i;
split(I,V,i,len,h);
i+=len;
len=0;
};
};
if(len) I[i-len]=-len;
};
for(i=0;i<oldsize+1;i++) I[V[i]]=i;
}
off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize)
{
off_t i;
for(i=0;(i<oldsize)&&(i<newsize);i++)
if(old[i]!=new[i]) break;
return i;
}
off_t search(off_t *I,u_char *old,off_t oldsize,
u_char *new,off_t newsize,off_t st,off_t en,off_t *pos)
{
off_t x,y;
if(en-st<2) {
x=matchlen(old+I[st],oldsize-I[st],new,newsize);
y=matchlen(old+I[en],oldsize-I[en],new,newsize);
if(x>y) {
*pos=I[st];
return x;
} else {
*pos=I[en];
return y;
}
};
x=st+(en-st)/2;
if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) {
return search(I,old,oldsize,new,newsize,x,en,pos);
} else {
return search(I,old,oldsize,new,newsize,st,x,pos);
};
}
void offtout(off_t x,u_char *buf)
{
off_t y;
if(x<0) y=-x; else y=x;
buf[0]=y%256;y-=buf[0];
y=y/256;buf[1]=y%256;y-=buf[1];
y=y/256;buf[2]=y%256;y-=buf[2];
y=y/256;buf[3]=y%256;y-=buf[3];
y=y/256;buf[4]=y%256;y-=buf[4];
y=y/256;buf[5]=y%256;y-=buf[5];
y=y/256;buf[6]=y%256;y-=buf[6];
y=y/256;buf[7]=y%256;
if(x<0) buf[7]|=0x80;
}
int main(int argc,char *argv[])
{
int fd,p[2];
pid_t pid;
u_char *old,*new;
off_t oldsize,newsize;
off_t *I,*V;
off_t scan,pos,len;
off_t lastscan,lastpos,lastoffset;
off_t oldscore,scsc;
off_t s,Sf,lenf,Sb,lenb;
off_t overlap,Ss,lens;
off_t i;
off_t dblen,eblen;
u_char *db,*eb;
u_char buf[8];
u_char header[32];
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
that we never try to malloc(0) and get a NULL pointer */
if(((fd=open(argv[1],O_RDONLY,0))<0) ||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
((old=malloc(oldsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,old,oldsize)!=oldsize) ||
(close(fd)==-1)) err(1,"%s",argv[1]);
if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) ||
((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL);
qsufsort(I,V,old,oldsize);
free(V);
/* Allocate newsize+1 bytes instead of newsize bytes to ensure
that we never try to malloc(0) and get a NULL pointer */
if(((fd=open(argv[2],O_RDONLY,0))<0) ||
((newsize=lseek(fd,0,SEEK_END))==-1) ||
((new=malloc(newsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,new,newsize)!=newsize) ||
(close(fd)==-1)) err(1,"%s",argv[2]);
if(((db=malloc(newsize+1))==NULL) ||
((eb=malloc(newsize+1))==NULL)) err(1,NULL);
dblen=0;
eblen=0;
if((fd=open(argv[3],O_CREAT|O_TRUNC|O_WRONLY,0666))<0)
err(1,"%s",argv[3]);
/* Header is
0 8 "BSDIFF40"
8 8 length of bzip2ed ctrl block
16 8 length of bzip2ed diff block
24 8 length of new file */
/* File is
0 32 Header
32 ?? Bzip2ed ctrl block
?? ?? Bzip2ed diff block
?? ?? Bzip2ed extra block */
memcpy(header,"BSDIFF40",8);
memset(header+8,0,24);
if(write(fd,header,32)!=32) err(1,"%s",argv[3]);
if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
if(pid==0) {
if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
(dup2(p[0],0)==-1) || (close(fd)==-1) ||
(close(p[0])==-1) || (close(p[1])==-1))
err(1,NULL);
execl(BZIP2,BZIP2,"-zc",NULL);
err(1,"%s",BZIP2);
};
if(close(p[0])==-1) err(1,NULL);
scan=0;len=0;
lastscan=0;lastpos=0;lastoffset=0;
while(scan<newsize) {
oldscore=0;
for(scsc=scan+=len;scan<newsize;scan++) {
len=search(I,old,oldsize,new+scan,newsize-scan,
0,oldsize,&pos);
for(;scsc<scan+len;scsc++)
if((scsc+lastoffset<oldsize) &&
(old[scsc+lastoffset] == new[scsc]))
oldscore++;
if(((len==oldscore) && (len!=0)) ||
(len>oldscore+8)) break;
if((scan+lastoffset<oldsize) &&
(old[scan+lastoffset] == new[scan]))
oldscore--;
};
if((len!=oldscore) || (scan==newsize)) {
s=0;Sf=0;lenf=0;
for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
if(old[lastpos+i]==new[lastscan+i]) s++;
i++;
if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
};
lenb=0;
if(scan<newsize) {
s=0;Sb=0;
for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
if(old[pos-i]==new[scan-i]) s++;
if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
};
};
if(lastscan+lenf>scan-lenb) {
overlap=(lastscan+lenf)-(scan-lenb);
s=0;Ss=0;lens=0;
for(i=0;i<overlap;i++) {
if(new[lastscan+lenf-overlap+i]==
old[lastpos+lenf-overlap+i]) s++;
if(new[scan-lenb+i]==
old[pos-lenb+i]) s--;
if(s>Ss) { Ss=s; lens=i+1; };
};
lenf+=lens-overlap;
lenb-=lens;
};
for(i=0;i<lenf;i++)
db[dblen+i]=new[lastscan+i]-old[lastpos+i];
for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
eb[eblen+i]=new[lastscan+lenf+i];
dblen+=lenf;
eblen+=(scan-lenb)-(lastscan+lenf);
offtout(lenf,buf);
if(write(p[1],buf,8)!=8) err(1,NULL);
offtout((scan-lenb)-(lastscan+lenf),buf);
if(write(p[1],buf,8)!=8) err(1,NULL);
offtout((pos-lenb)-(lastpos+lenf),buf);
if(write(p[1],buf,8)!=8) err(1,NULL);
lastscan=scan-lenb;
lastpos=pos-lenb;
lastoffset=pos-scan;
};
};
if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
if((len=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
offtout(len-32,buf);
if((lseek(fd,8,SEEK_SET)!=8) || (write(fd,buf,8)!=8))
err(1,"%s",argv[3]);
offtout(newsize,buf);
if((lseek(fd,24,SEEK_SET)!=24) || (write(fd,buf,8)!=8))
err(1,"%s",argv[3]);
if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
if(pid==0) {
if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
(dup2(p[0],0)==-1) || (close(fd)==-1) ||
(close(p[0])==-1) || (close(p[1])==-1))
err(1,NULL);
execl(BZIP2,BZIP2,"-zc",NULL);
err(1,"%s",BZIP2);
};
if(close(p[0])==-1) err(1,NULL);
if(write(p[1],db,dblen)!=dblen) err(1,NULL);
if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
if((newsize=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
offtout(newsize-len,buf);
if((lseek(fd,16,SEEK_SET)!=16) || (write(fd,buf,8)!=8))
err(1,"%s",argv[3]);
if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
if(pid==0) {
if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
(dup2(p[0],0)==-1) || (close(fd)==-1) ||
(close(p[0])==-1) || (close(p[1])==-1))
err(1,NULL);
execl(BZIP2,BZIP2,"-zc",NULL);
err(1,"%s",BZIP2);
};
if(close(p[0])==-1) err(1,NULL);
if(write(p[1],eb,eblen)!=eblen) err(1,NULL);
if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
if(close(fd)==-1) err(1,"%s",argv[3]);
free(db);
free(eb);
free(I);
free(old);
free(new);
return 0;
}
-32
View File
@@ -1,32 +0,0 @@
.Dd May 18, 2003
.Dt BSPATCH 1
.Os FreeBSD
.Sh NAME
.Nm bspatch
.Nd apply a patch built with bsdiff(1)
.Sh SYNOPSIS
.Nm
.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
.Sh DESCRIPTION
.Nm
generates
.Ao Ar newfile Ac
from
.Ao Ar oldfile Ac
and
.Ao Ar patchfile Ac
where
.Ao Ar patchfile Ac
is a binary patch built by bsdiff(1).
.Pp
.Nm
uses memory equal to the size of
.Ao Ar oldfile Ac
plus the size of
.Ao Ar newfile Ac ,
but can tolerate a very small working set without a dramatic loss
of performance.
.Sh SEE ALSO
.Xr bsdiff 1
.Sh AUTHORS
.An Colin Percival Aq cperciva@daemonology.net
-216
View File
@@ -1,216 +0,0 @@
/*
.c -- Binary patcher
Copyright 2003,2004 Colin Percival
For the terms under which this work may be distributed, please see
the adjoining file "LICENSE".
*/
#ifndef BZIP2
#define BZIP2 "/usr/bin/bzip2"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <err.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/wait.h>
ssize_t loopread(int d,void *buf,size_t nbytes)
{
ssize_t ptr,lenread;
for(ptr=0;ptr<nbytes;ptr+=lenread) {
lenread=read(d,buf+ptr,nbytes-ptr);
if(lenread==0) return ptr;
if(lenread==-1) return -1;
};
return ptr;
}
int bz2read(int fd,off_t offset,off_t len,char * fname,pid_t * pids)
{
int p0[2],p1[2];
u_char * data;
if((pipe(p0)==-1) || (pipe(p1)==-1)) err(1,NULL);
if((pids[0]=fork())==-1) err(1,NULL);
if(pids[0]==0) {
if(close(0) || close(1) || close(p0[0]) ||
close(p1[0]) || close(p1[1])) err(1,NULL);
if((data=malloc(len+1))==NULL) err(1,NULL);
if((pread(fd,data,len,offset)!=len) || close(fd))
err(1,"%s",fname);
if((write(p0[1],data,len)!=len) || close(p0[1]))
err(1,NULL);
free(data);
_exit(0);
};
if((pids[1]=fork())==-1) err(1,NULL);
if(pids[1]==0) {
if(close(0) || close(1) || close(p0[1]) ||
close(p1[0])) err(1,NULL);
if((dup2(p0[0],0)==-1) || close(p0[0])) err(1,NULL);
if((dup2(p1[1],1)==-1) || close(p1[1])) err(1,NULL);
if(close(fd)==-1) err(1,"%s",fname);
execl(BZIP2,BZIP2,"-dc",NULL);
err(1,"%s",BZIP2);
};
if(close(p0[0]) || close(p0[1]) || close(p1[1])) err(1,NULL);
return p1[0];
}
off_t offtin(u_char *buf)
{
off_t y;
y=buf[7]&0x7F;
y=y*256;y+=buf[6];
y=y*256;y+=buf[5];
y=y*256;y+=buf[4];
y=y*256;y+=buf[3];
y=y*256;y+=buf[2];
y=y*256;y+=buf[1];
y=y*256;y+=buf[0];
if(buf[7]&0x80) y=-y;
return y;
}
int main(int argc,char * argv[])
{
int fd,ctrlpipe,diffpipe,extrapipe;
pid_t pids[6];
ssize_t patchsize,oldsize,newsize;
ssize_t bzctrllen,bzdatalen;
u_char header[32],buf[8];
int version=0;
u_char *old, *new;
off_t oldpos,newpos;
off_t ctrl[3];
off_t lenread;
off_t i;
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
if(((fd=open(argv[3],O_RDONLY,0))<0) ||
((patchsize=lseek(fd,0,SEEK_END))==-1) ||
(lseek(fd,0,SEEK_SET)!=0)) err(1,"%s",argv[3]);
if(patchsize<32) errx(1,"Corrupt patch\n");
/*
Ok, this is going to be messy. There are two different patch
formats which we need to support.
The old format (pre-4.0) is:
0 8 "QSUFDIFF" or "BSDIFF30"
8 8 X
16 8 Y
24 8 sizeof(newfile)
32 X bzip2(control block)
32+X Y bzip2(data block)
with control block a set of pairs (x,y) meaning "seek forward
in oldfile by y bytes, and add the next x bytes to x bytes from
the data block".
The new format (4.0) is:
0 8 "BSDIFF40"
8 8 X
16 8 Y
24 8 sizeof(newfile)
32 X bzip2(control block)
32+X Y bzip2(diff block)
32+X+Y ??? bzip2(extra block)
with control block a set of triples (x,y,z) meaning "add x bytes
from oldfile to x bytes from the diff block; copy y bytes from the
extra block; seek forwards in oldfile by z bytes".
*/
if(read(fd,header,32)!=32) err(1,"%s",argv[3]);
if(memcmp(header,"QSUFDIFF",8)==0) version=1;
if(memcmp(header,"BSDIFF30",8)==0) version=1;
if(memcmp(header,"BSDIFF40",8)==0) version=2;
if(!version) errx(1,"Corrupt patch\n");
bzctrllen=offtin(header+8);
bzdatalen=offtin(header+16);
newsize=offtin(header+24);
if((bzctrllen<0) || (bzdatalen<0) || (newsize<0) ||
((version==1) && (32+bzctrllen+bzdatalen!=patchsize)))
errx(1,"Corrupt patch\n");
ctrlpipe=bz2read(fd,32,bzctrllen,argv[3],pids);
diffpipe=bz2read(fd,32+bzctrllen,bzdatalen,argv[3],pids+2);
if(version==2) {
extrapipe=bz2read(fd,32+bzctrllen+bzdatalen,
patchsize-(32+bzctrllen+bzdatalen),argv[3],pids+4);
};
if(close(fd)==-1) err(1,"%s",argv[3]);
if(((fd=open(argv[1],O_RDONLY,0))<0) ||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
((old=malloc(oldsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,old,oldsize)!=oldsize) ||
(close(fd)==-1)) err(1,"%s",argv[1]);
if((new=malloc(newsize+1))==NULL) err(1,NULL);
oldpos=0;newpos=0;
while(newpos<newsize) {
for(i=0;i<=version;i++) {
if((lenread=loopread(ctrlpipe,buf,8))<0) err(1,NULL);
if(lenread<8) errx(1,"Corrupt patch\n");
ctrl[i]=offtin(buf);
};
if(version==1) oldpos+=ctrl[1];
if(newpos+ctrl[0]>newsize) errx(1,"Corrupt patch\n");
if((lenread=loopread(diffpipe,new+newpos,ctrl[0]))<0)
err(1,NULL);
if(lenread!=ctrl[0]) errx(1,"Corrupt patch\n");
for(i=0;i<ctrl[0];i++)
if((oldpos+i>=0) && (oldpos+i<oldsize))
new[newpos+i]+=old[oldpos+i];
newpos+=ctrl[0];
oldpos+=ctrl[0];
if(version==2) {
if(newpos+ctrl[1]>newsize) errx(1,"Corrupt patch\n");
if((lenread=loopread(extrapipe,new+newpos,ctrl[1]))<0)
err(1,NULL);
if(lenread!=ctrl[1]) errx(1,"Corrupt patch\n");
newpos+=ctrl[1];
oldpos+=ctrl[2];
};
};
if(loopread(ctrlpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
if(loopread(diffpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
if(version==2)
if(loopread(extrapipe,buf,1)!=0) errx(1,"Corrupt patch\n");
if(close(ctrlpipe) || close(diffpipe) ||
((version==2) && close(extrapipe)))
err(1,NULL);
for(i=0;i<(version+1)*2;i++) waitpid(pids[i],NULL,0);
if(((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) ||
(write(fd,new,newsize)!=newsize) || (close(fd)==-1))
err(1,"%s",argv[2]);
free(new);
free(old);
return 0;
}
+1 -1
View File
@@ -29,7 +29,7 @@ lexer-tab.c lexer-tab.h: lexer.l
nixexpr-ast.cc nixexpr-ast.hh: ../aterm-helper.pl nixexpr-ast.def
$(perl) ../aterm-helper.pl nixexpr-ast.hh nixexpr-ast.cc < nixexpr-ast.def
nixexpr.cc nixexpr.hh: nixexpr-ast.hh
nixexpr.hh: nixexpr-ast.hh
CLEANFILES =
+1 -1
View File
@@ -1,5 +1,5 @@
#include "nixexpr.hh"
#include "derivations.hh"
#include "storeexpr.hh"
#include "nixexpr-ast.hh"
+6
View File
@@ -89,5 +89,11 @@ void checkVarDefs(const ATermMap & def, Expr e);
/* Create an expression representing a boolean. */
Expr makeBool(bool b);
/* Create an expression representing a string. */
Expr makeString(const string & s);
/* Create an expression representing a path. */
Expr makePath(const Path & path);
#endif /* !__NIXEXPR_H */
-61
View File
@@ -75,65 +75,6 @@ int yyparse(yyscan_t scanner, ParseData * data);
}
static void checkAttrs(ATermMap & names, ATermList bnds)
{
for (ATermIterator i(bnds); i; ++i) {
ATerm name;
Expr e;
ATerm pos;
if (!matchBind(*i, name, e, pos)) abort(); /* can't happen */
if (names.get(name))
throw Error(format("duplicate attribute `%1%' at %2%")
% aterm2String(name) % showPos(pos));
names.set(name, name);
}
}
static void checkAttrSets(ATerm e)
{
ATermList formals;
ATerm body, pos;
if (matchFunction(e, formals, body, pos)) {
ATermMap names;
for (ATermIterator i(formals); i; ++i) {
ATerm name;
Expr deflt;
if (!matchNoDefFormal(*i, name) &&
!matchDefFormal(*i, name, deflt))
abort();
if (names.get(name))
throw Error(format("duplicate formal function argument `%1%' at %2%")
% aterm2String(name) % showPos(pos));
names.set(name, name);
}
}
ATermList bnds;
if (matchAttrs(e, bnds)) {
ATermMap names;
checkAttrs(names, bnds);
}
ATermList rbnds, nrbnds;
if (matchRec(e, rbnds, nrbnds)) {
ATermMap names;
checkAttrs(names, rbnds);
checkAttrs(names, nrbnds);
}
if (ATgetType(e) == AT_APPL) {
int arity = ATgetArity(ATgetAFun(e));
for (int i = 0; i < arity; ++i)
checkAttrSets(ATgetArgument(e, i));
}
else if (ATgetType(e) == AT_LIST)
for (ATermIterator i((ATermList) e); i; ++i)
checkAttrSets(*i);
}
static Expr parse(EvalState & state,
const char * text, const Path & path,
const Path & basePath)
@@ -155,8 +96,6 @@ static Expr parse(EvalState & state,
} catch (Error & e) {
throw Error(format("%1%, in `%2%'") % e.msg() % path);
}
checkAttrSets(data.result);
return data.result;
}
+118 -174
View File
@@ -1,4 +1,4 @@
#include "build.hh"
#include "normalise.hh"
#include "eval.hh"
#include "globals.hh"
#include "nixexpr-ast.hh"
@@ -16,62 +16,72 @@ static Expr primImport(EvalState & state, const ATermVector & args)
}
/* Returns the hash of a derivation modulo fixed-output
subderivations. A fixed-output derivation is a derivation with one
output (`out') for which an expected hash and hash algorithm are
specified (using the `outputHash' and `outputHashAlgo'
attributes). We don't want changes to such derivations to
propagate upwards through the dependency graph, changing output
paths everywhere.
For instance, if we change the url in a call to the `fetchurl'
function, we do not want to rebuild everything depending on it
(after all, (the hash of) the file being downloaded is unchanged).
So the *output paths* should not change. On the other hand, the
*derivation store expression paths* should change to reflect the
new dependency graph.
That's what this function does: it returns a hash which is just the
of the derivation ATerm, except that any input store expression
paths have been replaced by the result of a recursive call to this
function, and that for fixed-output derivations we return
(basically) its outputHash. */
static Hash hashDerivationModulo(EvalState & state, Derivation drv)
static PathSet storeExprRootsCached(EvalState & state, const Path & nePath)
{
/* Return a fixed hash for fixed-output derivations. */
if (drv.outputs.size() == 1) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
if (i->first == "out" &&
i->second.hash != "")
{
return hashString(htSHA256, "fixed:out:"
+ i->second.hashAlgo + ":"
+ i->second.hash + ":"
+ i->second.path);
}
DrvRoots::iterator i = state.drvRoots.find(nePath);
if (i != state.drvRoots.end())
return i->second;
else {
PathSet paths = storeExprRoots(nePath);
state.drvRoots[nePath] = paths;
return paths;
}
/* For other derivations, replace the inputs paths with recursive
calls to this function.*/
DerivationInputs inputs2;
for (DerivationInputs::iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i)
{
Hash h = state.drvHashes[i->first];
if (h.type == htUnknown) {
Derivation drv2 = derivationFromPath(i->first);
h = hashDerivationModulo(state, drv2);
state.drvHashes[i->first] = h;
}
inputs2[printHash(h)] = i->second;
}
drv.inputDrvs = inputs2;
return hashTerm(unparseDerivation(drv));
}
static void processBinding(EvalState & state, Expr e, Derivation & drv,
static Hash hashDerivation(EvalState & state, StoreExpr ne)
{
if (ne.type == StoreExpr::neDerivation) {
PathSet inputs2;
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); i++)
{
DrvHashes::iterator j = state.drvHashes.find(*i);
if (j == state.drvHashes.end())
throw Error(format("don't know expression `%1%'") % (string) *i);
inputs2.insert(j->second);
}
ne.derivation.inputs = inputs2;
}
return hashTerm(unparseStoreExpr(ne));
}
static Path copyAtom(EvalState & state, const Path & srcPath)
{
/* !!! should be cached */
Path dstPath(addToStore(srcPath));
ClosureElem elem;
StoreExpr ne;
ne.type = StoreExpr::neClosure;
ne.closure.roots.insert(dstPath);
ne.closure.elems[dstPath] = elem;
Hash drvHash = hashDerivation(state, ne);
Path drvPath = writeTerm(unparseStoreExpr(ne), "");
state.drvHashes[drvPath] = drvHash;
state.drvRoots[drvPath] = ne.closure.roots;
printMsg(lvlChatty, format("copied `%1%' -> closure `%2%'")
% srcPath % drvPath);
return drvPath;
}
static string addInput(EvalState & state,
Path & nePath, StoreExpr & ne)
{
PathSet paths = storeExprRootsCached(state, nePath);
if (paths.size() != 1) abort();
Path path = *(paths.begin());
ne.derivation.inputs.insert(nePath);
return path;
}
static void processBinding(EvalState & state, Expr e, StoreExpr & ne,
Strings & ss)
{
e = evalExpr(state, e);
@@ -94,61 +104,37 @@ static void processBinding(EvalState & state, Expr e, Derivation & drv,
else if (matchAttrs(e, es)) {
Expr a = queryAttr(e, "type");
if (a && evalString(state, a) == "derivation") {
a = queryAttr(e, "drvPath");
if (!a) throw Error("derivation name missing");
Path drvPath = evalPath(state, a);
a = queryAttr(e, "outPath");
if (!a) throw Error("output path missing");
/* !!! supports only single output path */
Path outPath = evalPath(state, a);
drv.inputDrvs[drvPath] = singleton<StringSet>("out");
ss.push_back(outPath);
}
else if (a && evalString(state, a) == "storePath") {
a = queryAttr(e, "drvHash");
if (!a) throw Error("derivation hash missing");
Hash drvHash = parseHash(evalString(state, a));
a = queryAttr(e, "outPath");
if (!a) throw Error("output path missing");
/* !!! supports only single output path */
Path outPath = evalPath(state, a);
PathSet drvRoots;
drvRoots.insert(evalPath(state, a));
state.drvHashes[drvPath] = drvHash;
state.drvRoots[drvPath] = drvRoots;
drv.inputSrcs.insert(outPath);
ss.push_back(outPath);
}
else throw Error("invalid derivation attribute");
ss.push_back(addInput(state, drvPath, ne));
} else
throw Error("invalid derivation attribute");
}
else if (matchPath(e, s)) {
Path srcPath(canonPath(aterm2String(s)));
if (isStorePath(srcPath)) {
printMsg(lvlChatty, format("using store path `%1%' as source")
% srcPath);
drv.inputSrcs.insert(srcPath);
ss.push_back(srcPath);
}
else {
if (isDerivation(srcPath))
throw Error(format("file names are not allowed to end in `%1%'")
% drvExtension);
Path dstPath(addToStore(srcPath));
printMsg(lvlChatty, format("copied source `%1%' -> `%2%'")
% srcPath % dstPath);
drv.inputSrcs.insert(dstPath);
ss.push_back(dstPath);
}
Path drvPath = copyAtom(state, aterm2String(s));
ss.push_back(addInput(state, drvPath, ne));
}
else if (matchList(e, es)) {
for (ATermIterator i(es); i; ++i) {
startNest(nest, lvlVomit, format("processing list element"));
processBinding(state, evalExpr(state, *i), drv, ss);
processBinding(state, evalExpr(state, *i), ne, ss);
}
}
@@ -156,7 +142,7 @@ static void processBinding(EvalState & state, Expr e, Derivation & drv,
else if (matchSubPath(e, e1, e2)) {
Strings ss2;
processBinding(state, evalExpr(state, e1), drv, ss2);
processBinding(state, evalExpr(state, e1), ne, ss2);
if (ss2.size() != 1)
throw Error("left-hand side of `~' operator cannot be a list");
e2 = evalExpr(state, e2);
@@ -198,13 +184,12 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
queryAllAttrs(args, attrs, true);
/* Build the derivation expression by processing the attributes. */
Derivation drv;
StoreExpr ne;
ne.type = StoreExpr::neDerivation;
string drvName;
string outputHash;
string outputHashAlgo;
bool outputHashRecursive = false;
Hash outHash;
bool outHashGiven = false;
for (ATermIterator i(attrs.keys()); i; ++i) {
string key = aterm2String(*i);
@@ -216,7 +201,7 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
Strings ss;
try {
processBinding(state, value, drv, ss);
processBinding(state, value, ne, ss);
} catch (Error & e) {
throw Error(format("while processing the derivation attribute `%1%' at %2%:\n%3%")
% key % showPos(pos) % e.msg());
@@ -226,98 +211,68 @@ static Expr primDerivation(EvalState & state, const ATermVector & _args)
command-line arguments to the builder. */
if (key == "args") {
for (Strings::iterator i = ss.begin(); i != ss.end(); ++i)
drv.args.push_back(*i);
ne.derivation.args.push_back(*i);
}
/* All other attributes are passed to the builder through the
environment. */
else {
string s = concatStrings(ss);
drv.env[key] = s;
if (key == "builder") drv.builder = s;
else if (key == "system") drv.platform = s;
ne.derivation.env[key] = s;
if (key == "builder") ne.derivation.builder = s;
else if (key == "system") ne.derivation.platform = s;
else if (key == "name") drvName = s;
else if (key == "outputHash") outputHash = s;
else if (key == "outputHashAlgo") outputHashAlgo = s;
else if (key == "outputHashMode") {
if (s == "recursive") outputHashRecursive = true;
else if (s == "flat") outputHashRecursive = false;
else throw Error(format("invalid value `%1%' for `outputHashMode' attribute") % s);
else if (key == "id") {
outHash = parseHash(s);
outHashGiven = true;
}
}
}
/* Do we have all required attributes? */
if (drv.builder == "")
if (ne.derivation.builder == "")
throw Error("required attribute `builder' missing");
if (drv.platform == "")
if (ne.derivation.platform == "")
throw Error("required attribute `system' missing");
if (drvName == "")
throw Error("required attribute `name' missing");
/* If an output hash was given, check it. */
if (outputHash == "")
outputHashAlgo = "";
else {
HashType ht = parseHashType(outputHashAlgo);
if (ht == htUnknown)
throw Error(format("unknown hash algorithm `%1%'") % outputHashAlgo);
Hash h;
if (outputHash.size() == Hash(ht).hashSize * 2)
/* hexadecimal representation */
h = parseHash(ht, outputHash);
else
/* base-32 representation */
h = parseHash32(ht, outputHash);
string s = outputHash;
outputHash = printHash(h);
if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo;
}
/* Check the derivation name. It shouldn't contain whitespace,
but we are conservative here: we check whether only
alphanumerics and some other characters appear. */
checkStoreName(drvName);
if (isDerivation(drvName))
throw Error(format("derivation names are not allowed to end in `%1%'")
% drvExtension);
/* !!! the name should not end in the derivation extension (.drv).
Likewise for sources. */
/* Construct the "masked" derivation store expression, which is
the final one except that in the list of outputs, the output
paths are empty, and the corresponding environment variables
have an empty value. This ensures that changes in the set of
output names do get reflected in the hash. */
drv.env["out"] = "";
drv.outputs["out"] =
DerivationOutput("", outputHashAlgo, outputHash);
string validChars = "+-._?=";
for (string::iterator i = drvName.begin(); i != drvName.end(); ++i)
if (!((*i >= 'A' && *i <= 'Z') ||
(*i >= 'a' && *i <= 'z') ||
(*i >= '0' && *i <= '9') ||
validChars.find(*i) != string::npos))
{
throw Error(format("invalid character `%1%' in derivation name `%2%'")
% *i % drvName);
}
/* Use the masked derivation expression to compute the output
path. */
Path outPath = makeStorePath("output:out",
hashDerivationModulo(state, drv), drvName);
/* Construct the final derivation store expression. */
drv.env["out"] = outPath;
drv.outputs["out"] =
DerivationOutput(outPath, outputHashAlgo, outputHash);
/* Determine the output path by hashing the Nix expression with no
outputs to produce a unique but deterministic path name for
this derivation. */
if (!outHashGiven) outHash = hashDerivation(state, ne);
Path outPath = canonPath(nixStore + "/" +
((string) outHash).c_str() + "-" + drvName);
ne.derivation.env["out"] = outPath;
ne.derivation.outputs.insert(outPath);
/* Write the resulting term into the Nix store directory. */
Path drvPath = writeDerivation(drv, drvName);
Hash drvHash = outHashGiven
? hashString((string) outHash + outPath)
: hashDerivation(state, ne);
Path drvPath = writeTerm(unparseStoreExpr(ne), "-d-" + drvName);
printMsg(lvlChatty, format("instantiated `%1%' -> `%2%'")
% drvName % drvPath);
/* Optimisation, but required in read-only mode! because in that
case we don't actually write store expressions, so we can't
read them later. */
state.drvHashes[drvPath] = hashDerivationModulo(state, drv);
/* !!! assumes a single output */
attrs.set("outPath", makeAttrRHS(makePath(toATerm(outPath)), makeNoPos()));
attrs.set("drvPath", makeAttrRHS(makePath(toATerm(drvPath)), makeNoPos()));
attrs.set("drvHash",
makeAttrRHS(makeStr(toATerm((string) drvHash)), makeNoPos()));
attrs.set("type", makeAttrRHS(makeStr(toATerm("derivation")), makeNoPos()));
return makeAttrs(attrs);
@@ -357,21 +312,21 @@ static Expr primFalse(EvalState & state, const ATermVector & args)
/* Return the null value. */
static Expr primNull(EvalState & state, const ATermVector & args)
Expr primNull(EvalState & state, const ATermVector & args)
{
return makeNull();
}
/* Determine whether the argument is the null value. */
static Expr primIsNull(EvalState & state, const ATermVector & args)
Expr primIsNull(EvalState & state, const ATermVector & args)
{
return makeBool(matchNull(evalExpr(state, args[0])));
}
/* Apply a function to every element of a list. */
static Expr primMap(EvalState & state, const ATermVector & args)
Expr primMap(EvalState & state, const ATermVector & args)
{
Expr fun = evalExpr(state, args[0]);
Expr list = evalExpr(state, args[1]);
@@ -388,22 +343,11 @@ static Expr primMap(EvalState & state, const ATermVector & args)
}
/* Return a string constant representing the current platform. Note!
that differs between platforms, so Nix expressions using
`__currentSystem' can evaluate to different values on different
platforms. */
static Expr primCurrentSystem(EvalState & state, const ATermVector & args)
{
return makeStr(toATerm(thisSystem));
}
void EvalState::addPrimOps()
{
addPrimOp("true", 0, primTrue);
addPrimOp("false", 0, primFalse);
addPrimOp("null", 0, primNull);
addPrimOp("__currentSystem", 0, primCurrentSystem);
addPrimOp("import", 1, primImport);
addPrimOp("derivation", 1, primDerivation);
-1
View File
@@ -7,6 +7,5 @@ AM_CXXFLAGS = \
-DNIX_DATA_DIR=\"$(datadir)\" \
-DNIX_STATE_DIR=\"$(localstatedir)/nix\" \
-DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \
-DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \
-DNIX_VERSION=\"$(VERSION)\" \
-I.. ${aterm_include} -I../libutil -I../libstore
+6 -43
View File
@@ -13,7 +13,6 @@ extern "C" {
}
#include "globals.hh"
#include "gc.hh"
#include "shared.hh"
#include "config.h"
@@ -31,28 +30,6 @@ void sigintHandler(int signo)
}
Path makeRootName(const Path & gcRoot, int & counter)
{
counter++;
if (counter == 1)
return gcRoot;
else
return (format("%1%-%2%") % gcRoot % counter).str();
}
void printGCWarning()
{
static bool warned = false;
if (!warned) {
printMsg(lvlInfo,
"warning: you did not specify `--add-root'; "
"the result might be removed by the garbage collector");
warned = true;
}
}
void setLogType(string lt)
{
if (lt == "pretty") logType = ltPretty;
@@ -78,16 +55,7 @@ void checkStoreNotSymlink(Path path)
}
struct RemoveTempRoots
{
~RemoveTempRoots()
{
removeTempRoots();
}
};
void initDerivationsHelpers();
void initStoreExprHelpers();
/* Initialize and reorder arguments, then call the actual argument
@@ -101,12 +69,11 @@ static void initAndRun(int argc, char * * argv)
}
/* Setup Nix paths. */
nixStore = canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)));
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
nixStore = getEnv("NIX_STORE_DIR", canonPath(NIX_STORE_DIR));
nixDataDir = getEnv("NIX_DATA_DIR", canonPath(NIX_DATA_DIR));
nixLogDir = getEnv("NIX_LOG_DIR", canonPath(NIX_LOG_DIR));
nixStateDir = getEnv("NIX_STATE_DIR", canonPath(NIX_STATE_DIR));
nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
/* Check that the store directory and its parent are not
symlinks. */
@@ -138,7 +105,7 @@ static void initAndRun(int argc, char * * argv)
if (lt != "") setLogType(lt);
/* ATerm stuff. !!! find a better place to put this */
initDerivationsHelpers();
initStoreExprHelpers();
/* Put the arguments in a vector. */
Strings args, remaining;
@@ -204,10 +171,6 @@ static void initAndRun(int argc, char * * argv)
else remaining.push_back(arg);
}
/* Automatically clean up the temporary roots file when we
exit. */
RemoveTempRoots removeTempRoots; /* unused variable - don't remove */
run(remaining);
}
-4
View File
@@ -17,10 +17,6 @@ void run(Strings args);
/* Should print a help message to stdout and return. */
void printHelp();
/* Ugh. No better place to put this. */
Path makeRootName(const Path & gcRoot, int & counter);
void printGCWarning();
extern string programId;
+7 -7
View File
@@ -1,18 +1,18 @@
noinst_LIBRARIES = libstore.a
libstore_a_SOURCES = \
store.cc store.hh derivations.cc derivations.hh \
build.cc misc.cc build.hh \
store.cc store.hh storeexpr.cc storeexpr.hh \
normalise.cc misc.cc normalise.hh \
globals.cc globals.hh db.cc db.hh \
references.cc references.hh pathlocks.cc pathlocks.hh \
gc.cc gc.hh derivations-ast.hh
gc.cc gc.hh storeexpr-ast.hh
EXTRA_DIST = derivations-ast.def derivations-ast.cc
EXTRA_DIST = storeexpr-ast.def storeexpr-ast.cc
AM_CXXFLAGS = -Wall \
-I.. ${bdb_include} ${aterm_include} -I../libutil
derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def
$(perl) ../aterm-helper.pl derivations-ast.hh derivations-ast.cc < derivations-ast.def
storeexpr-ast.cc storeexpr-ast.hh: ../aterm-helper.pl storeexpr-ast.def
$(perl) ../aterm-helper.pl storeexpr-ast.hh storeexpr-ast.cc < storeexpr-ast.def
derivations.cc store.cc: derivations-ast.hh
storeexpr.cc: storeexpr-ast.hh
-36
View File
@@ -1,36 +0,0 @@
#ifndef __BUILD_H
#define __BUILD_H
#include "derivations.hh"
/* Ensure that the output paths of the derivation are valid. If they
are already valid, this is a no-op. Otherwise, validity can
be reached in two ways. First, if the output paths have
substitutes, then those can be used. Second, the output paths can
be created by running the builder, after recursively building any
sub-derivations. */
void buildDerivations(const PathSet & drvPaths);
/* Ensure that a path is valid. If it is not currently valid, it may
be made valid by running a substitute (if defined for the path). */
void ensurePath(const Path & storePath);
/* Read a derivation, after ensuring its existence through
ensurePath(). */
Derivation derivationFromPath(const Path & drvPath);
/* Place in `paths' the set of all store paths in the file system
closure of `storePath'; that is, all paths than can be directly or
indirectly reached from it. `paths' is not cleared. If
`flipDirection' is true, the set of paths that can reach
`storePath' is returned; that is, the closures under the `referers'
relation instead of the `references' relation is returned. */
void computeFSClosure(const Path & storePath,
PathSet & paths, bool flipDirection = false);
/* Return the path corresponding to the output identifier `id' in the
given derivation. */
Path findOutput(const Derivation & drv, string id);
#endif /* !__BUILD_H */
+8 -16
View File
@@ -33,9 +33,11 @@ Transaction::Transaction()
Transaction::Transaction(Database & db)
: txn(0)
{
begin(db);
db.requireEnv();
try {
db.env->txn_begin(0, &txn, 0);
} catch (DbException e) { rethrow(e); }
}
@@ -45,16 +47,6 @@ Transaction::~Transaction()
}
void Transaction::begin(Database & db)
{
assert(txn == 0);
db.requireEnv();
try {
db.env->txn_begin(0, &txn, 0);
} catch (DbException e) { rethrow(e); }
}
void Transaction::commit()
{
if (!txn) throw Error("commit called on null transaction");
@@ -90,7 +82,7 @@ void Transaction::moveTo(Transaction & t)
void Database::requireEnv()
{
checkInterrupt();
if (!env) throw Error("database environment is not open "
if (!env)throw Error("database environment is not open "
"(maybe you don't have sufficient permission?)");
}
@@ -194,9 +186,9 @@ void Database::open(const string & path)
number, then run db_recover on the database to remove the
existing DB environment (since changes only take effect on
new environments). */
env->set_lk_max_locks(10000);
env->set_lk_max_lockers(10000);
env->set_lk_max_objects(10000);
env->set_lk_max_locks(4000);
env->set_lk_max_lockers(4000);
env->set_lk_max_objects(4000);
env->set_lk_detect(DB_LOCK_DEFAULT);
/* Dangerous, probably, but from the docs it *seems* that BDB
-1
View File
@@ -27,7 +27,6 @@ public:
Transaction(Database & _db);
~Transaction();
void begin(Database & db);
void abort();
void commit();
-10
View File
@@ -1,10 +0,0 @@
init initDerivationsHelpers
Derive | ATermList ATermList ATermList string string ATermList ATermList | ATerm |
| string string | ATerm | EnvBinding |
| string ATermList | ATerm | DerivationInput |
| string string string string | ATerm | DerivationOutput |
Closure | ATermList ATermList | ATerm | OldClosure |
| string ATermList | ATerm | OldClosureElem |
-169
View File
@@ -1,169 +0,0 @@
#include "derivations.hh"
#include "globals.hh"
#include "store.hh"
#include "derivations-ast.hh"
#include "derivations-ast.cc"
Hash hashTerm(ATerm t)
{
return hashString(htSHA256, atPrint(t));
}
Path writeDerivation(const Derivation & drv, const string & name)
{
PathSet references;
references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i)
references.insert(i->first);
/* Note that the outputs of a derivation are *not* references
(that can be missing (of course) and should not necessarily be
held during a garbage collection). */
return addTextToStore(name + drvExtension,
atPrint(unparseDerivation(drv)), references);
}
static void checkPath(const string & s)
{
if (s.size() == 0 || s[0] != '/')
throw Error(format("bad path `%1%' in derivation") % s);
}
static void parseStrings(ATermList paths, StringSet & out, bool arePaths)
{
for (ATermIterator i(paths); i; ++i) {
if (ATgetType(*i) != AT_APPL)
throw badTerm("not a path", *i);
string s = aterm2String(*i);
if (arePaths) checkPath(s);
out.insert(s);
}
}
void throwBadDrv(ATerm t)
{
throw badTerm("not a valid derivation", t);
}
Derivation parseDerivation(ATerm t)
{
Derivation drv;
ATermList outs, inDrvs, inSrcs, args, bnds;
ATerm builder, platform;
if (!matchDerive(t, outs, inDrvs, inSrcs, platform, builder, args, bnds))
throwBadDrv(t);
for (ATermIterator i(outs); i; ++i) {
ATerm id, path, hashAlgo, hash;
if (!matchDerivationOutput(*i, id, path, hashAlgo, hash))
throwBadDrv(t);
DerivationOutput out;
out.path = aterm2String(path);
checkPath(out.path);
out.hashAlgo = aterm2String(hashAlgo);
out.hash = aterm2String(hash);
drv.outputs[aterm2String(id)] = out;
}
for (ATermIterator i(inDrvs); i; ++i) {
ATerm drvPath;
ATermList ids;
if (!matchDerivationInput(*i, drvPath, ids))
throwBadDrv(t);
Path drvPath2 = aterm2String(drvPath);
checkPath(drvPath2);
StringSet ids2;
parseStrings(ids, ids2, false);
drv.inputDrvs[drvPath2] = ids2;
}
parseStrings(inSrcs, drv.inputSrcs, true);
drv.builder = aterm2String(builder);
drv.platform = aterm2String(platform);
for (ATermIterator i(args); i; ++i) {
if (ATgetType(*i) != AT_APPL)
throw badTerm("string expected", *i);
drv.args.push_back(aterm2String(*i));
}
for (ATermIterator i(bnds); i; ++i) {
ATerm s1, s2;
if (!matchEnvBinding(*i, s1, s2))
throw badTerm("tuple of strings expected", *i);
drv.env[aterm2String(s1)] = aterm2String(s2);
}
return drv;
}
static ATermList unparseStrings(const StringSet & paths)
{
ATermList l = ATempty;
for (PathSet::const_iterator i = paths.begin();
i != paths.end(); ++i)
l = ATinsert(l, toATerm(*i));
return ATreverse(l);
}
ATerm unparseDerivation(const Derivation & drv)
{
ATermList outputs = ATempty;
for (DerivationOutputs::const_iterator i = drv.outputs.begin();
i != drv.outputs.end(); ++i)
outputs = ATinsert(outputs,
makeDerivationOutput(
toATerm(i->first),
toATerm(i->second.path),
toATerm(i->second.hashAlgo),
toATerm(i->second.hash)));
ATermList inDrvs = ATempty;
for (DerivationInputs::const_iterator i = drv.inputDrvs.begin();
i != drv.inputDrvs.end(); ++i)
inDrvs = ATinsert(inDrvs,
makeDerivationInput(
toATerm(i->first),
unparseStrings(i->second)));
ATermList args = ATempty;
for (Strings::const_iterator i = drv.args.begin();
i != drv.args.end(); ++i)
args = ATinsert(args, toATerm(*i));
ATermList env = ATempty;
for (StringPairs::const_iterator i = drv.env.begin();
i != drv.env.end(); ++i)
env = ATinsert(env,
makeEnvBinding(
toATerm(i->first),
toATerm(i->second)));
return makeDerive(
ATreverse(outputs),
ATreverse(inDrvs),
unparseStrings(drv.inputSrcs),
toATerm(drv.platform),
toATerm(drv.builder),
ATreverse(args),
ATreverse(env));
}
bool isDerivation(const string & fileName)
{
return
fileName.size() >= drvExtension.size() &&
string(fileName, fileName.size() - drvExtension.size()) == drvExtension;
}
-67
View File
@@ -1,67 +0,0 @@
#ifndef __DERIVATIONS_H
#define __DERIVATIONS_H
#include "aterm.hh"
#include "store.hh"
/* Extension of derivations in the Nix store. */
const string drvExtension = ".drv";
/* Abstract syntax of derivations. */
struct DerivationOutput
{
Path path;
string hashAlgo; /* hash used for expected hash computation */
string hash; /* expected hash, may be null */
DerivationOutput()
{
}
DerivationOutput(Path path, string hashAlgo, string hash)
{
this->path = path;
this->hashAlgo = hashAlgo;
this->hash = hash;
}
};
typedef map<string, DerivationOutput> DerivationOutputs;
/* For inputs that are sub-derivations, we specify exactly which
output IDs we are interested in. */
typedef map<Path, StringSet> DerivationInputs;
typedef map<string, string> StringPairs;
struct Derivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
DerivationInputs inputDrvs; /* inputs that are sub-derivations */
PathSet inputSrcs; /* inputs that are sources */
string platform;
Path builder;
Strings args;
StringPairs env;
};
/* Hash an aterm. */
Hash hashTerm(ATerm t);
/* Write a derivation to the Nix store, and return its path. */
Path writeDerivation(const Derivation & drv, const string & name);
/* Parse a derivation. */
Derivation parseDerivation(ATerm t);
/* Parse a derivation. */
ATerm unparseDerivation(const Derivation & drv);
/* Check whether a file name ends with the extensions for
derivations. */
bool isDerivation(const string & fileName);
#endif /* !__DERIVATIONS_H */
+68 -427
View File
@@ -1,457 +1,98 @@
#include "normalise.hh"
#include "globals.hh"
#include "gc.hh"
#include "build.hh"
#include "pathlocks.hh"
#include <boost/shared_ptr.hpp>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
static string gcLockName = "gc.lock";
static string tempRootsDir = "temproots";
static string gcRootsDir = "gcroots";
/* Acquire the global GC lock. This is used to prevent new Nix
processes from starting after the temporary root files have been
read. To be precise: when they try to create a new temporary root
file, they will block until the garbage collector has finished /
yielded the GC lock. */
static int openGCLock(LockType lockType)
void followLivePaths(Path nePath, PathSet & live)
{
Path fnGCLock = (format("%1%/%2%")
% nixStateDir % gcLockName).str();
debug(format("acquiring global GC lock `%1%'") % fnGCLock);
/* Just to be sure, canonicalise the path. It is important to do
this here and in findDeadPath() to ensure that a live path is
not mistaken for a dead path due to some non-canonical
representation. */
nePath = canonPath(nePath);
AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT, 0600);
if (fdGCLock == -1)
throw SysError(format("opening global GC lock `%1%'") % fnGCLock);
if (live.find(nePath) != live.end()) return;
live.insert(nePath);
lockFile(fdGCLock, lockType, true);
startNest(nest, lvlDebug, format("following `%1%'") % nePath);
assertStorePath(nePath);
/* !!! Restrict read permission on the GC root. Otherwise any
process that can open the file for reading can DoS the
collector. */
if (isValidPath(nePath)) {
/* !!! should make sure that no substitutes are used */
StoreExpr ne = storeExprFromPath(nePath);
/* !!! painfully similar to requisitesWorker() */
if (ne.type == StoreExpr::neClosure)
for (ClosureElems::iterator i = ne.closure.elems.begin();
i != ne.closure.elems.end(); ++i)
{
Path p = canonPath(i->first);
if (live.find(p) == live.end()) {
debug(format("found live `%1%'") % p);
assertStorePath(p);
live.insert(p);
}
}
return fdGCLock.borrow();
}
else if (ne.type == StoreExpr::neDerivation)
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); ++i)
followLivePaths(*i, live);
void createSymlink(const Path & link, const Path & target, bool careful)
{
/* Create directories up to `gcRoot'. */
createDirs(dirOf(link));
/* Remove the old symlink. */
if (pathExists(link)) {
if (careful && (!isLink(link) || !isInStore(readLink(link))))
throw Error(format("cannot create symlink `%1%'; already exists") % link);
unlink(link.c_str());
}
/* And create the new own. */
if (symlink(target.c_str(), link.c_str()) == -1)
throw SysError(format("symlinking `%1%' to `%2%'")
% link % target);
}
Path addPermRoot(const Path & _storePath, const Path & _gcRoot,
bool indirect)
{
Path storePath(canonPath(_storePath));
Path gcRoot(canonPath(_gcRoot));
assertStorePath(storePath);
/* Grab the global GC root. This prevents the set of permanent
roots from increasing while a GC is in progress. */
AutoCloseFD fdGCLock = openGCLock(ltRead);
if (indirect) {
string hash = printHash32(hashString(htSHA1, gcRoot));
Path realRoot = canonPath((format("%1%/%2%/auto/%3%")
% nixStateDir % gcRootsDir % hash).str());
else abort();
{
SwitchToOriginalUser sw;
createSymlink(gcRoot, storePath, true);
}
createSymlink(realRoot, gcRoot, false);
}
else {
Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/")
throw Error(format(
"path `%1%' is not a valid garbage collector root; "
"it's not in the directory `%2%'")
% gcRoot % rootsDir);
createSymlink(gcRoot, storePath, false);
}
return gcRoot;
Path nfPath;
if (querySuccessor(nePath, nfPath))
followLivePaths(nfPath, live);
}
/* The file to which we write our temporary roots. */
static Path fnTempRoots;
static AutoCloseFD fdTempRoots;
void addTempRoot(const Path & path)
PathSet findLivePaths(const Paths & roots)
{
/* Create the temporary roots file for this process. */
if (fdTempRoots == -1) {
PathSet live;
while (1) {
Path dir = (format("%1%/%2%") % nixStateDir % tempRootsDir).str();
createDirs(dir);
fnTempRoots = (format("%1%/%2%")
% dir % getpid()).str();
startNest(nest, lvlDebug, "finding live paths");
AutoCloseFD fdGCLock = openGCLock(ltRead);
fdTempRoots = open(fnTempRoots.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0600);
if (fdTempRoots == -1)
throw SysError(format("opening temporary roots file `%1%'") % fnTempRoots);
for (Paths::const_iterator i = roots.begin(); i != roots.end(); ++i)
followLivePaths(*i, live);
fdGCLock.close();
debug(format("acquiring read lock on `%1%'") % fnTempRoots);
lockFile(fdTempRoots, ltRead, true);
return live;
}
/* Check whether the garbage collector didn't get in our
way. */
PathSet findDeadPaths(const PathSet & live, time_t minAge)
{
PathSet dead;
startNest(nest, lvlDebug, "finding dead paths");
time_t now = time(0);
Strings storeNames = readDirectory(nixStore);
for (Strings::iterator i = storeNames.begin(); i != storeNames.end(); ++i) {
Path p = canonPath(nixStore + "/" + *i);
if (minAge > 0) {
struct stat st;
if (fstat(fdTempRoots, &st) == -1)
throw SysError(format("statting `%1%'") % fnTempRoots);
if (st.st_size == 0) break;
/* The garbage collector deleted this file before we could
get a lock. (It won't delete the file after we get a
lock.) Try again. */
if (lstat(p.c_str(), &st) != 0)
throw SysError(format("obtaining information about `%1%'") % p);
if (st.st_atime + minAge >= now) continue;
}
}
/* Upgrade the lock to a write lock. This will cause us to block
if the garbage collector is holding our lock. */
debug(format("acquiring write lock on `%1%'") % fnTempRoots);
lockFile(fdTempRoots, ltWrite, true);
string s = path + '\0';
writeFull(fdTempRoots, (const unsigned char *) s.c_str(), s.size());
/* Downgrade to a read lock. */
debug(format("downgrading to read lock on `%1%'") % fnTempRoots);
lockFile(fdTempRoots, ltRead, true);
}
void removeTempRoots()
{
if (fdTempRoots != -1) {
fdTempRoots.close();
unlink(fnTempRoots.c_str());
}
}
typedef shared_ptr<AutoCloseFD> FDPtr;
typedef list<FDPtr> FDs;
static void readTempRoots(PathSet & tempRoots, FDs & fds)
{
/* Read the `temproots' directory for per-process temporary root
files. */
Strings tempRootFiles = readDirectory(
(format("%1%/%2%") % nixStateDir % tempRootsDir).str());
for (Strings::iterator i = tempRootFiles.begin();
i != tempRootFiles.end(); ++i)
{
Path path = (format("%1%/%2%/%3%") % nixStateDir % tempRootsDir % *i).str();
debug(format("reading temporary root file `%1%'") % path);
FDPtr fd(new AutoCloseFD(open(path.c_str(), O_RDWR, 0666)));
if (*fd == -1) {
/* It's okay if the file has disappeared. */
if (errno == ENOENT) continue;
throw SysError(format("opening temporary roots file `%1%'") % path);
}
/* Try to acquire a write lock without blocking. This can
only succeed if the owning process has died. In that case
we don't care about its temporary roots. */
if (lockFile(*fd, ltWrite, false)) {
printMsg(lvlError, format("removing stale temporary roots file `%1%'")
% path);
unlink(path.c_str());
writeFull(*fd, (const unsigned char *) "d", 1);
continue;
}
/* Acquire a read lock. This will prevent the owning process
from upgrading to a write lock, therefore it will block in
addTempRoot(). */
debug(format("waiting for read lock on `%1%'") % path);
lockFile(*fd, ltRead, true);
/* Read the entire file. */
string contents = readFile(*fd);
/* Extract the roots. */
unsigned int pos = 0, end;
while ((end = contents.find((char) 0, pos)) != string::npos) {
Path root(contents, pos, end - pos);
debug(format("got temporary root `%1%'") % root);
assertStorePath(root);
tempRoots.insert(root);
pos = end + 1;
}
fds.push_back(fd); /* keep open */
}
}
static void findRoots(const Path & path, bool recurseSymlinks,
PathSet & roots)
{
struct stat st;
if (lstat(path.c_str(), &st) == -1)
throw SysError(format("statting `%1%'") % path);
printMsg(lvlVomit, format("looking at `%1%'") % path);
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
findRoots(path + "/" + *i, recurseSymlinks, roots);
}
else if (S_ISLNK(st.st_mode)) {
string target = readLink(path);
Path target2 = absPath(target, dirOf(path));
if (isInStore(target2)) {
debug(format("found root `%1%' in `%2%'")
% target2 % path);
roots.insert(toStorePath(target2));
}
else if (recurseSymlinks) {
if (pathExists(target2))
findRoots(target2, false, roots);
else {
printMsg(lvlInfo, format("removing stale link from `%1%' to `%2%'") % path % target2);
/* Note that we only delete when recursing, i.e., when
we are still in the `gcroots' tree. We never
delete stuff outside that tree. */
unlink(path.c_str());
}
}
}
}
static void dfsVisit(const PathSet & paths, const Path & path,
PathSet & visited, Paths & sorted)
{
if (visited.find(path) != visited.end()) return;
visited.insert(path);
PathSet references;
if (isValidPath(path))
queryReferences(noTxn, path, references);
for (PathSet::iterator i = references.begin();
i != references.end(); ++i)
/* Don't traverse into paths that don't exist. That can
happen due to substitutes for non-existent paths. */
if (*i != path && paths.find(*i) != paths.end())
dfsVisit(paths, *i, visited, sorted);
sorted.push_front(path);
}
static Paths topoSort(const PathSet & paths)
{
Paths sorted;
PathSet visited;
for (PathSet::const_iterator i = paths.begin(); i != paths.end(); ++i)
dfsVisit(paths, *i, visited, sorted);
return sorted;
}
void collectGarbage(GCAction action, PathSet & result)
{
result.clear();
bool gcKeepOutputs =
queryBoolSetting("gc-keep-outputs", false);
bool gcKeepDerivations =
queryBoolSetting("gc-keep-derivations", true);
/* Acquire the global GC root. This prevents
a) New roots from being added.
b) Processes from creating new temporary root files. */
AutoCloseFD fdGCLock = openGCLock(ltWrite);
/* Find the roots. Since we've grabbed the GC lock, the set of
permanent roots cannot increase now. */
Path rootsDir = canonPath((format("%1%/%2%") % nixStateDir % gcRootsDir).str());
PathSet roots;
findRoots(rootsDir, true, roots);
if (action == gcReturnRoots) {
result = roots;
return;
}
/* Determine the live paths which is just the closure of the
roots under the `references' relation. */
PathSet livePaths;
for (PathSet::const_iterator i = roots.begin(); i != roots.end(); ++i)
computeFSClosure(canonPath(*i), livePaths);
if (gcKeepDerivations) {
for (PathSet::iterator i = livePaths.begin();
i != livePaths.end(); ++i)
{
/* Note that the deriver need not be valid (e.g., if we
previously ran the collector with `gcKeepDerivations'
turned off). */
Path deriver = queryDeriver(noTxn, *i);
if (deriver != "" && isValidPath(deriver))
computeFSClosure(deriver, livePaths);
}
}
if (gcKeepOutputs) {
/* Hmz, identical to storePathRequisites in nix-store. */
for (PathSet::iterator i = livePaths.begin();
i != livePaths.end(); ++i)
if (isDerivation(*i)) {
Derivation drv = derivationFromPath(*i);
for (DerivationOutputs::iterator j = drv.outputs.begin();
j != drv.outputs.end(); ++j)
if (isValidPath(j->second.path))
computeFSClosure(j->second.path, livePaths);
}
}
if (action == gcReturnLive) {
result = livePaths;
return;
}
/* Read the temporary roots. This acquires read locks on all
per-process temporary root files. So after this point no paths
can be added to the set of temporary roots. */
PathSet tempRoots;
FDs fds;
readTempRoots(tempRoots, fds);
/* Close the temporary roots. Note that we *cannot* do this in
readTempRoots(), because there we may not have all locks yet,
meaning that an invalid path can become valid (and thus add to
the references graph) after we have added it to the closure
(and computeFSClosure() assumes that the presence of a path
means that it has already been closed). */
PathSet tempRootsClosed;
for (PathSet::iterator i = tempRoots.begin(); i != tempRoots.end(); ++i)
if (isValidPath(*i))
computeFSClosure(*i, tempRootsClosed);
else
tempRootsClosed.insert(*i);
/* For testing - see tests/gc-concurrent.sh. */
if (getenv("NIX_DEBUG_GC_WAIT"))
sleep(2);
/* After this point the set of roots or temporary roots cannot
increase, since we hold locks on everything. So everything
that is not currently in in `livePaths' or `tempRootsClosed'
can be deleted. */
/* Read the Nix store directory to find all currently existing
paths. */
Paths storePaths = readDirectory(nixStore);
PathSet storePaths2;
for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i)
storePaths2.insert(canonPath(nixStore + "/" + *i));
/* Topologically sort them under the `referers' relation. That
is, a < b iff a is in referers(b). This gives us the order in
which things can be deleted safely. */
/* !!! when we have multiple output paths per derivation, this
will not work anymore because we get cycles. */
storePaths = topoSort(storePaths2);
/* Try to delete store paths in the topologically sorted order. */
for (Paths::iterator i = storePaths.begin(); i != storePaths.end(); ++i) {
debug(format("considering deletion of `%1%'") % *i);
if (livePaths.find(*i) != livePaths.end()) {
debug(format("live path `%1%'") % *i);
continue;
}
if (tempRootsClosed.find(*i) != tempRootsClosed.end()) {
debug(format("temporary root `%1%'") % *i);
continue;
}
debug(format("dead path `%1%'") % *i);
result.insert(*i);
AutoCloseFD fdLock;
if (action == gcDeleteDead) {
/* Only delete a lock file if we can acquire a write lock
on it. That means that it's either stale, or the
process that created it hasn't locked it yet. In the
latter case the other process will detect that we
deleted the lock, and retry (see pathlocks.cc). */
if (i->size() >= 5 && string(*i, i->size() - 5) == ".lock") {
fdLock = open(i->c_str(), O_RDWR);
if (fdLock == -1) {
if (errno == ENOENT) continue;
throw SysError(format("opening lock file `%1%'") % *i);
}
if (!lockFile(fdLock, ltWrite, false)) {
debug(format("skipping active lock `%1%'") % *i);
continue;
}
}
printMsg(lvlInfo, format("deleting `%1%'") % *i);
/* Okay, it's safe to delete. */
deleteFromStore(*i);
if (fdLock != -1)
/* Write token to stale (deleted) lock file. */
writeFull(fdLock, (const unsigned char *) "d", 1);
}
if (live.find(p) == live.end()) {
debug(format("dead path `%1%'") % p);
dead.insert(p);
} else
debug(format("live path `%1%'") % p);
}
return dead;
}
+17 -31
View File
@@ -1,40 +1,26 @@
#ifndef __GC_H
#define __GC_H
#include "util.hh"
#include "storeexpr.hh"
/* Garbage collector operation. */
typedef enum {
gcReturnRoots,
gcReturnLive,
gcReturnDead,
gcDeleteDead,
} GCAction;
/* Determine the set of "live" store paths, given a set of root store
expressions. The live store paths are those that are reachable
from the roots. The roots are reachable by definition. Any path
mentioned in a reachable store expression is also reachable. If a
derivation store expression is reachable, then its successor (if it
exists) if also reachable. It is not an error for store
expressions not to exist (since this can happen on derivation store
expressions, for instance, due to the substitute mechanism), but
successor links are followed even for non-existant derivations. */
PathSet findLivePaths(const Paths & roots);
/* If `action' is set to `gcReturnRoots', find and return the set of
roots for the garbage collector. These are the store paths
symlinked to in the `gcroots' directory. If `action' is
`gcReturnLive', return the set of paths reachable from (i.e. in the
closure of) the roots. If `action' is `gcReturnDead', return the
set of paths not reachable from the roots. If `action' is
`gcDeleteDead', actually delete the latter set. */
void collectGarbage(GCAction action, PathSet & result);
/* Register a temporary GC root. This root will automatically
disappear when this process exits. WARNING: this function should
not be called inside a BDB transaction, otherwise we can
deadlock. */
void addTempRoot(const Path & path);
/* Remove the temporary roots file for this process. Any temporary
root becomes garbage after this point unless it has been registered
as a (permanent) root. */
void removeTempRoots();
/* Register a permanent GC root. */
Path addPermRoot(const Path & storePath, const Path & gcRoot,
bool indirect);
/* Given a set of "live" store paths, determine the set of "dead"
store paths (which are simply all store paths that are not in the
live set). The value `minAge' specifies the minimum age in seconds
for an unreachable file to be considered dead (0 meaning that any
unreachable file is dead). */
PathSet findDeadPaths(const PathSet & live, time_t minAge);
#endif /* !__GC_H */
-60
View File
@@ -1,14 +1,10 @@
#include "globals.hh"
#include <map>
string nixStore = "/UNINIT";
string nixDataDir = "/UNINIT";
string nixLogDir = "/UNINIT";
string nixStateDir = "/UNINIT";
string nixDBPath = "/UNINIT";
string nixConfDir = "/UNINIT";
bool keepFailed = false;
@@ -21,59 +17,3 @@ Verbosity buildVerbosity = lvlInfo;
unsigned int maxBuildJobs = 1;
bool readOnlyMode = false;
static bool settingsRead = false;
static map<string, string> settings;
static void readSettings()
{
Path settingsFile = (format("%1%/%2%") % nixConfDir % "nix.conf").str();
if (!pathExists(settingsFile)) return;
string contents = readFile(settingsFile);
unsigned int pos = 0;
while (pos < contents.size()) {
string line;
while (pos < contents.size() && contents[pos] != '\n')
line += contents[pos++];
pos++;
unsigned int hash = line.find('#');
if (hash != string::npos)
line = string(line, 0, hash);
if (line.find_first_not_of(" ") == string::npos) continue;
istringstream is(line);
string name, sep, value;
is >> name >> sep >> value;
if (sep != "=" || !is)
throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile);
settings[name] = value;
};
settingsRead = true;
}
string querySetting(const string & name, const string & def)
{
if (!settingsRead) readSettings();
map<string, string>::iterator i = settings.find(name);
return i == settings.end() ? def : i->second;
}
bool queryBoolSetting(const string & name, bool def)
{
string value = querySetting(name, def ? "true" : "false");
if (value == "true") return true;
else if (value == "false") return false;
else throw Error(format("configuration option `%1%' should be either `true' or `false', not `%2%'")
% name % value);
}
-10
View File
@@ -23,11 +23,6 @@ extern string nixStateDir;
/* nixDBPath is the path name of our Berkeley DB environment. */
extern string nixDBPath;
/* nixConfDir is the directory where configuration files are
stored. */
extern string nixConfDir;
/* Misc. global flags. */
@@ -53,9 +48,4 @@ extern unsigned int maxBuildJobs;
extern bool readOnlyMode;
string querySetting(const string & name, const string & def);
bool queryBoolSetting(const string & name, bool def);
#endif /* !__GLOBALS_H */
+58 -24
View File
@@ -1,38 +1,72 @@
#include "build.hh"
#include "normalise.hh"
Derivation derivationFromPath(const Path & drvPath)
StoreExpr storeExprFromPath(const Path & path)
{
assertStorePath(drvPath);
ensurePath(drvPath);
ATerm t = ATreadFromNamedFile(drvPath.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % drvPath);
return parseDerivation(t);
assertStorePath(path);
ensurePath(path);
ATerm t = ATreadFromNamedFile(path.c_str());
if (!t) throw Error(format("cannot read aterm from `%1%'") % path);
return parseStoreExpr(t);
}
void computeFSClosure(const Path & storePath,
PathSet & paths, bool flipDirection)
PathSet storeExprRoots(const Path & nePath)
{
if (paths.find(storePath) != paths.end()) return;
paths.insert(storePath);
PathSet paths;
PathSet references;
if (flipDirection)
queryReferers(noTxn, storePath, references);
else
queryReferences(noTxn, storePath, references);
StoreExpr ne = storeExprFromPath(nePath);
for (PathSet::iterator i = references.begin();
i != references.end(); ++i)
computeFSClosure(*i, paths, flipDirection);
if (ne.type == StoreExpr::neClosure)
paths.insert(ne.closure.roots.begin(), ne.closure.roots.end());
else if (ne.type == StoreExpr::neDerivation)
paths.insert(ne.derivation.outputs.begin(),
ne.derivation.outputs.end());
else abort();
return paths;
}
Path findOutput(const Derivation & drv, string id)
static void requisitesWorker(const Path & nePath,
bool includeExprs, bool includeSuccessors,
PathSet & paths, PathSet & doneSet)
{
for (DerivationOutputs::const_iterator i = drv.outputs.begin();
i != drv.outputs.end(); ++i)
if (i->first == id) return i->second.path;
throw Error(format("derivation has no output `%1%'") % id);
checkInterrupt();
if (doneSet.find(nePath) != doneSet.end()) return;
doneSet.insert(nePath);
StoreExpr ne = storeExprFromPath(nePath);
if (ne.type == StoreExpr::neClosure)
for (ClosureElems::iterator i = ne.closure.elems.begin();
i != ne.closure.elems.end(); ++i)
paths.insert(i->first);
else if (ne.type == StoreExpr::neDerivation)
for (PathSet::iterator i = ne.derivation.inputs.begin();
i != ne.derivation.inputs.end(); ++i)
requisitesWorker(*i,
includeExprs, includeSuccessors, paths, doneSet);
else abort();
if (includeExprs) paths.insert(nePath);
Path nfPath;
if (includeSuccessors && querySuccessor(nePath, nfPath))
requisitesWorker(nfPath, includeExprs, includeSuccessors,
paths, doneSet);
}
PathSet storeExprRequisites(const Path & nePath,
bool includeExprs, bool includeSuccessors)
{
PathSet paths;
PathSet doneSet;
requisitesWorker(nePath, includeExprs, includeSuccessors,
paths, doneSet);
return paths;
}
File diff suppressed because it is too large. Load diff
+44
View File
@@ -0,0 +1,44 @@
#ifndef __NORMALISE_H
#define __NORMALISE_H
#include "storeexpr.hh"
/* Normalise a store expression. That is, if the expression is a
derivation, a path containing an equivalent closure expression is
returned. This requires that the derivation is performed, unless a
successor is known. */
Path normaliseStoreExpr(const Path & nePath);
/* Realise a store expression. If the expression is a derivation, it
is first normalised into a closure. The closure is then realised
in the file system (i.e., it is ensured that each path in the
closure exists in the file system, if necessary by using the
substitute mechanism). Returns the normal form of the expression
(i.e., its closure expression). */
Path realiseStoreExpr(const Path & nePath);
/* Ensure that a path exists, possibly by instantiating it by
realising a substitute. */
void ensurePath(const Path & path);
/* Read a store expression, after ensuring its existence through
ensurePath(). */
StoreExpr storeExprFromPath(const Path & path);
/* Get the list of root (output) paths of the given store
expression. */
PathSet storeExprRoots(const Path & nePath);
/* Get the list of paths that are required to realise the given store
expression. For a derive expression, this is the union of
requisites of the inputs; for a closure expression, it is the path
of each element in the closure. If `includeExprs' is true, include
the paths of the store expressions themselves. If
`includeSuccessors' is true, include the requisites of
successors. */
PathSet storeExprRequisites(const Path & nePath,
bool includeExprs, bool includeSuccessors);
#endif /* !__NORMALISE_H */
+19 -39
View File
@@ -61,7 +61,7 @@ PathLocks::PathLocks(const PathSet & paths)
void PathLocks::lockPaths(const PathSet & _paths)
{
/* May be called only once! */
assert(fds.empty());
assert(this->paths.empty());
/* Note that `fds' is built incrementally so that the destructor
will only release those locks that we have already acquired. */
@@ -83,38 +83,20 @@ void PathLocks::lockPaths(const PathSet & _paths)
debug(format("already holding lock on `%1%'") % lockPath);
continue;
}
AutoCloseFD fd;
while (1) {
/* Open/create the lock file. */
fd = open(lockPath.c_str(), O_WRONLY | O_CREAT, 0666);
if (fd == -1)
throw SysError(format("opening lock file `%1%'") % lockPath);
/* Open/create the lock file. */
int fd = open(lockPath.c_str(), O_WRONLY | O_CREAT, 0666);
if (fd == -1)
throw SysError(format("opening lock file `%1%'") % lockPath);
/* Acquire an exclusive lock. */
lockFile(fd, ltWrite, true);
fds.push_back(fd);
this->paths.push_back(lockPath);
debug(format("lock acquired on `%1%'") % lockPath);
/* Acquire an exclusive lock. */
lockFile(fd, ltWrite, true);
/* Check that the lock file hasn't become stale (i.e.,
hasn't been unlinked). */
struct stat st;
if (fstat(fd, &st) == -1)
throw SysError(format("statting lock file `%1%'") % lockPath);
if (st.st_size != 0)
/* This lock file has been unlinked, so we're holding
a lock on a deleted file. This means that other
processes may create and acquire a lock on
`lockPath', and proceed. So we must retry. */
debug(format("open lock file `%1%' has become stale") % lockPath);
else
break;
}
debug(format("lock acquired on `%1%'") % lockPath);
/* Use borrow so that the descriptor isn't closed. */
fds.push_back(FDPair(fd.borrow(), lockPath));
lockedPaths.insert(lockPath);
}
}
@@ -122,21 +104,19 @@ void PathLocks::lockPaths(const PathSet & _paths)
PathLocks::~PathLocks()
{
for (list<FDPair>::iterator i = fds.begin(); i != fds.end(); i++) {
for (list<int>::iterator i = fds.begin(); i != fds.end(); i++)
if (close(*i) != 0) throw SysError("closing fd");
for (Paths::iterator i = paths.begin(); i != paths.end(); i++) {
checkInterrupt();
if (deletePaths) {
/* Write a (meaningless) token to the file to indicate to
other processes waiting on this lock that the lock is
stale (deleted). */
unlink(i->second.c_str());
writeFull(i->first, (const unsigned char *) "d", 1);
/* This is not safe in general! */
unlink(i->c_str());
/* Note that the result of unlink() is ignored; removing
the lock file is an optimisation, not a necessity. */
}
lockedPaths.erase(i->second);
if (close(i->first) == -1)
printMsg(lvlError,
format("error (ignored): cannot close lock file on `%1%'") % i->second);
debug(format("lock released on `%1%'") % i->second);
lockedPaths.erase(*i);
debug(format("lock released on `%1%'") % *i);
}
}
+2 -2
View File
@@ -12,8 +12,8 @@ bool lockFile(int fd, LockType lockType, bool wait);
class PathLocks
{
private:
typedef pair<int, Path> FDPair;
list<FDPair> fds;
list<int> fds;
Paths paths;
bool deletePaths;
public:
+4 -8
View File
@@ -34,8 +34,6 @@ void checkPath(const string & path,
{
checkInterrupt();
debug(format("checking `%1%'") % path);
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
@@ -50,6 +48,8 @@ void checkPath(const string & path,
else if (S_ISREG(st.st_mode)) {
debug(format("checking `%1%'") % path);
AutoCloseFD fd = open(path.c_str(), O_RDONLY);
if (fd == -1) throw SysError(format("opening file `%1%'") % path);
@@ -81,12 +81,8 @@ Strings filterReferences(const string & path, const Strings & paths)
for (Strings::const_iterator i = paths.begin();
i != paths.end(); i++)
{
string baseName = baseNameOf(*i);
unsigned int pos = baseName.find('-');
if (pos == string::npos)
throw Error(format("bad reference `%1%'") % *i);
string s = string(baseName, 0, pos);
// parseHash(htSHA256, s);
string s = string(baseNameOf(*i), 0, 32);
parseHash(s);
ids.push_back(s);
backMap[s] = *i;
}
+284 -662
View File
File diff suppressed because it is too large. Load diff
+38 -98
View File
@@ -9,17 +9,15 @@
using namespace std;
const int nixSchemaVersion = 2;
/* A substitute is a program invocation that constructs some store
path (typically by fetching it from somewhere, e.g., from the
network). */
struct Substitute
{
/* The derivation that built this store path (empty if none). */
Path deriver;
{
/* Store expression to be normalised and realised in order to
obtain `program'. */
Path storeExpr;
/* Program to be executed to create the store path. Must be in
the output path of `storeExpr'. */
Path program;
@@ -46,115 +44,57 @@ void createStoreTransaction(Transaction & txn);
/* Copy a path recursively. */
void copyPath(const Path & src, const Path & dst);
/* Register a successor. This function accepts a transaction handle
so that it can be enclosed in an atomic operation with calls to
registerValidPath(). This must be atomic, since if we register a
successor for a derivation without registering the paths built in
the derivation, we have a successor with dangling pointers, and if
we do it in reverse order, we can get an obstructed build (since to
rebuild the successor, the outputs paths must not exist). */
void registerSuccessor(const Transaction & txn,
const Path & srcPath, const Path & sucPath);
/* Remove a successor mapping. */
void unregisterSuccessor(const Path & srcPath);
/* Return the predecessors of the Nix expression stored at the given
path. */
bool querySuccessor(const Path & srcPath, Path & sucPath);
/* Return the predecessors of the Nix expression stored at the given
path. */
Paths queryPredecessors(const Path & sucPath);
/* Register a substitute. */
void registerSubstitute(const Transaction & txn,
const Path & srcPath, const Substitute & sub);
typedef list<pair<Path, Substitute> > SubstitutePairs;
void registerSubstitutes(const Transaction & txn,
const SubstitutePairs & subPairs);
/* Return the substitutes for the given path. */
Substitutes querySubstitutes(const Transaction & txn, const Path & srcPath);
/* Return the substitutes expression for the given path. */
Substitutes querySubstitutes(const Path & srcPath);
/* Deregister all substitutes. */
void clearSubstitutes();
/* Register the validity of a path, i.e., that `path' exists, that the
paths referenced by it exists, and in the case of an output path of
a derivation, that it has been produced by a succesful execution of
the derivation (or something equivalent). Also register the hash
of the file system contents of the path. The hash must be a
SHA-256 hash. */
void registerValidPath(const Transaction & txn,
const Path & path, const Hash & hash, const PathSet & references,
const Path & deriver);
struct ValidPathInfo
{
Path path;
Path deriver;
Hash hash;
PathSet references;
};
typedef list<ValidPathInfo> ValidPathInfos;
void registerValidPaths(const Transaction & txn,
const ValidPathInfos & infos);
/* Register the validity of a path. */
void registerValidPath(const Transaction & txn, const Path & path);
/* Throw an exception if `path' is not directly in the Nix store. */
void assertStorePath(const Path & path);
bool isInStore(const Path & path);
bool isStorePath(const Path & path);
void checkStoreName(const string & name);
/* Chop off the parts after the top-level store name, e.g.,
/nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
Path toStorePath(const Path & path);
/* "Fix", or canonicalise, the meta-data of the files in a store path
after it has been built. In particular:
- the last modification date on each file is set to 0 (i.e.,
00:00:00 1/1/1970 UTC)
- the permissions are set of 444 or 555 (i.e., read-only with or
without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're
in a setuid Nix installation. */
void canonicalisePathMetaData(const Path & path);
/* Checks whether a path is valid. */
bool isValidPathTxn(const Transaction & txn, const Path & path);
bool isValidPath(const Path & path);
/* Queries the hash of a valid path. */
Hash queryPathHash(const Path & path);
/* Sets the set of outgoing FS references for a store path. Use with
care! */
void setReferences(const Transaction & txn, const Path & storePath,
const PathSet & references);
/* Queries the set of outgoing FS references for a store path. The
result is not cleared. */
void queryReferences(const Transaction & txn,
const Path & storePath, PathSet & references);
/* Queries the set of incoming FS references for a store path. The
result is not cleared. */
void queryReferers(const Transaction & txn,
const Path & storePath, PathSet & referers);
/* Sets the deriver of a store path. Use with care! */
void setDeriver(const Transaction & txn, const Path & storePath,
const Path & deriver);
/* Query the deriver of a store path. Return the empty string if no
deriver has been set. */
Path queryDeriver(const Transaction & txn, const Path & storePath);
/* Constructs a unique store path name. */
Path makeStorePath(const string & type,
const Hash & hash, const string & suffix);
/* Copy the contents of a path to the store and register the validity
the resulting path. The resulting path is returned. */
Path addToStore(const Path & srcPath);
/* Like addToStore(), but for pre-adding the outputs of fixed-output
derivations. */
Path addToStoreFixed(bool recursive, string hashAlgo, const Path & srcPath);
Path makeFixedOutputPath(bool recursive,
string hashAlgo, Hash hash, string name);
/* Like addToStore, but the contents written to the output path is a
regular file containing the given string. */
Path addTextToStore(const string & suffix, const string & s,
const PathSet & references);
/* Like addToStore, but the path of the output is given, and the
contents written to the output path is a regular file containing
the given string. */
void addTextToStore(const Path & dstPath, const string & s);
/* Delete a value from the nixStore directory. */
void deleteFromStore(const Path & path);
void verifyStore(bool checkContents);
void verifyStore();
#endif /* !__STORE_H */
+7
View File
@@ -0,0 +1,7 @@
init initStoreExprHelpers
Closure | ATermList ATermList | ATerm |
Derive | ATermList ATermList string string ATermList ATermList | ATerm |
| string string | ATerm | EnvBinding |
| string ATermList | ATerm | ClosureElem |
Loaded 100 of 161 files, more files were not shown because too many files have changed in this diff. Show more