Compare commits

..
Author SHA1 Message Date
Eelco Dolstra 3e95bb49be * Move this back temporarily. 2004-11-14 15:04:51 +00:00
Eelco Dolstra 84cc89b585 * Tag release 0.6. 2004-11-14 12:58:20 +00:00
Eelco Dolstra ac76ee5088 * Mark stable release. 2004-11-14 12:09:31 +00:00
Eelco Dolstra f8e4f731d4 * Release branch for Nix 0.6. 2004-11-14 12:04:21 +00:00
41 changed files with 344 additions and 1975 deletions

No files matched your search

-1
View File
@@ -29,7 +29,6 @@ init-state:
rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(prefix)/store
$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
# $(bindir)/nix-store --init
else
init-state:
-22
View File
@@ -1,25 +1,3 @@
Version 0.7
* Binary patching. When upgrading components using pre-built binaries
(through nix-pull / nix-channel), Nix can automatically download and
apply binary patches to already installed components instead of full
downloads. Patching is "smart": if there is a *sequence* of patches
to an installed component, Nix will use it. Patches are currently
generated automatically between Nixpkgs (pre-)releases.
* Simplifications to the substitute mechanism.
* Nix-pull now stores downloaded manifests in /nix/var/nix/manifests.
* Metadata on files in the Nix store is canonicalised after builds:
the last-modified timestamp is set to 0 (00:00:00 1/1/1970), the
mode is set to 0444 or 0555 (readable and possibly executable by
all; setuid/setgid bits are dropped), and the group is set to the
default. This ensures that the result of a build and an
installation through a substitute is the same; and that timestamp
dependencies are revealed.
Version 0.6
Major changes include the following:
+2 -2
View File
@@ -1,4 +1,4 @@
AC_INIT(nix, "0.7")
AC_INIT(nix, "0.6")
AC_CONFIG_SRCDIR(README)
AC_CONFIG_AUX_DIR(config)
AM_INIT_AUTOMAKE
@@ -189,13 +189,13 @@ AC_CONFIG_FILES([Makefile
src/nix-instantiate/Makefile
src/nix-env/Makefile
src/log2xml/Makefile
src/bsdiff-4.2/Makefile
scripts/Makefile
corepkgs/Makefile
corepkgs/fetchurl/Makefile
corepkgs/nar/Makefile
corepkgs/buildenv/Makefile
corepkgs/channels/Makefile
corepkgs/nix-pull/Makefile
doc/Makefile
doc/manual/Makefile
misc/Makefile
+1 -1
View File
@@ -1 +1 @@
SUBDIRS = fetchurl nar buildenv channels
SUBDIRS = fetchurl nar buildenv channels nix-pull
+4 -8
View File
@@ -6,14 +6,10 @@ export PATH=/bin:/usr/bin
echo "packing $path into $out..."
mkdir $out
dst=$out/$(basename $path).nar.bz2
@bindir@/nix-store --dump "$path" > tmp
@bindir@/nix-store --dump "$path" | @bzip2@ > $dst
@bzip2@ < tmp > $dst
if test "${PIPESTATUS[0]}" != "0"; then exit 1; fi
narHash=$(md5sum -b tmp | cut -c1-32)
md5=$(md5sum -b $dst | cut -c1-32)
if test $? != 0; then exit 1; fi
echo $narHash > $out/nar-hash
narbz2Hash=$(md5sum -b $dst | cut -c1-32)
if test $? != 0; then exit 1; fi
echo $narbz2Hash > $out/narbz2-hash
echo $md5 > $out/md5
+11
View File
@@ -0,0 +1,11 @@
all-local: builder.sh
install-exec-local:
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs
$(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
$(INSTALL_DATA) default.nix $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
$(INSTALL_PROGRAM) builder.sh $(DESTDIR)$(datadir)/nix/corepkgs/nix-pull
include ../../substitute.mk
EXTRA_DIST = default.nix builder.sh.in
+34
View File
@@ -0,0 +1,34 @@
#! @shell@ -e
export PATH=/bin:/usr/bin
mkdir $out
cat > $out/fetch <<EOF
#! @shell@ -e
export PATH=/bin:/usr/bin
echo "downloading \$2..."
export PRINT_PATH=1
result=(\$(@bindir@/nix-prefetch-url \$2))
hash=\${result[0]}
path=\${result[1]}
if test "\$hash" != "\$3"; then
echo "hash is \$hash, expected \$3"
exit 1
fi
echo "unpacking into \$1..."
if ! @bunzip2@ < "\$path" | @bindir@/nix-store --restore "\$1"; then
exit 1
fi
exit 0
EOF
chmod +x $out/fetch
+7
View File
@@ -0,0 +1,7 @@
{system}:
derivation {
name = "nix-pull";
builder = ./builder.sh;
inherit system;
}
+1 -8
View File
@@ -75,7 +75,7 @@ be adapted easily to achieve similar policies.</para></listitem>
<listitem><para>Nix component builds aim to be <quote>pure</quote>;
that is, unaffected by anything other than the declared dependencies.
This means that if a component was built successfully once, it can be
This means that if a component was built succesfully once, it can be
rebuilt again on another machine and the result will be the same. We
cannot <emphasis>guarantee</emphasis> this (e.g., if the build depends
on the time-of-day), but Nix (and the tools in the Nix Packages
@@ -113,13 +113,6 @@ software deployment also apply here: for instance, the ability
trivially to have multiple configurations at the same time, or the
ability to do rollbacks.</para></listitem>
<listitem><para>Nix can efficiently upgrade between different versions
of a component through <emphasis>binary patching</emphasis>. If
patches are available on a server, and you try to install a new
version of some component, Nix will automatically apply a patch (or
sequence of patches), if available, to transform the installed
component into the new version.</para></listitem>
</itemizedlist>
</para>
+2 -13
View File
@@ -575,7 +575,7 @@ builder becomes even shorter:
<programlisting>
. $stdenv/setup
genericBuild</programlisting>
genericBuilder</programlisting>
In fact, <varname>mkDerivation</varname> provides a default builder
that looks exactly like that, so it is actually possible to omit the
@@ -1253,17 +1253,6 @@ command-line argument. See <xref linkend='sec-standard-environment'
performed by looking for the hash parts of file names of the
inputs.</para></listitem>
<listitem><para>After the build, Nix sets the last-modified
timestamp on all files in the build result to 0 (00:00:00 1/1/1970
UTC), sets the group to the default group, and sets the mode of the
file to 0444 or 0555 (i.e., read-only, with execute permission
enabled if the file was originally executable). Note that possible
<literal>setuid</literal> and <literal>setgid</literal> bits are
cleared. Setuid and setgid programs are not currently supported by
Nix. This is because the Nix archives used in deployment have no
concept of ownership information, and because it makes the build
result dependent on the user performing the build.</para></listitem>
</itemizedlist>
</para>
@@ -1382,7 +1371,7 @@ following:
</para>
<para>The <filename>setup</filename> script also exports a function
called <function>genericBuild</function> that knows how to build
called <function>genericBuilder</function> that knows how to build
typical Autoconf-style components. It can be customised to perform
builds for any type of component. It is advisable to use
<function>genericBuild</function> since it provides facilities that
+3 -3
View File
@@ -34,12 +34,12 @@ endif
# CWI ATerm
ATERM = aterm-2.3.1
ATERM = aterm-2.2
$(ATERM).tar.gz:
@echo "Nix requires the CWI ATerm library to build."
@echo "Please download version 2.3.1 from"
@echo " http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.3.1.tar.gz"
@echo "Please download version 2.2 from"
@echo " http://www.cwi.nl/projects/MetaEnv/aterm/aterm-2.2.tar.gz"
@echo "and place it in the externals/ directory."
false
+6 -6
View File
@@ -4,15 +4,16 @@ bin_SCRIPTS = nix-collect-garbage \
noinst_SCRIPTS = nix-profile.sh
nix-pull nix-push: readmanifest.pm download-using-manifests.pl
nix-pull nix-push: readmanifest.pm
install-exec-local: readmanifest.pm download-using-manifests.pl
install-exec-local: readmanifest.pm
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/profile.d
$(INSTALL_PROGRAM) nix-profile.sh $(DESTDIR)$(sysconfdir)/profile.d/nix.sh
$(INSTALL) -d $(DESTDIR)$(libexecdir)/nix
$(INSTALL_DATA) readmanifest.pm $(DESTDIR)$(libexecdir)/nix
$(INSTALL_PROGRAM) download-using-manifests.pl $(DESTDIR)$(libexecdir)/nix
$(INSTALL) -d $(DESTDIR)$(sysconfdir)/nix
# !!! don't overwrite local modifications
$(INSTALL_DATA) prebuilts.conf $(DESTDIR)$(sysconfdir)/nix/prebuilts.conf
include ../substitute.mk
@@ -20,6 +21,5 @@ EXTRA_DIST = nix-collect-garbage.in \
nix-pull.in nix-push.in nix-profile.sh.in \
nix-prefetch-url.in nix-install-package.in \
nix-channel.in \
readmanifest.pm.in \
nix-build.in \
download-using-manifests.pl.in
prebuilts.conf readmanifest.pm.in \
nix-build.in
-246
View File
@@ -1,246 +0,0 @@
#! @perl@ -w -I@libexecdir@/nix
use strict;
use readmanifest;
my $manifestDir = "@localstatedir@/nix/manifests";
my $logFile = "@localstatedir@/log/nix/downloads";
open LOGFILE, ">>$logFile" or die "cannot open log file $logFile";
# Check the arguments.
die unless scalar @ARGV == 1;
my $targetPath = $ARGV[0];
my $date = `date`;
chomp $date;
print LOGFILE "$$ get $targetPath $date\n";
print "\n*** Trying to download/patch `$targetPath'\n";
# Load all manifests.
my %narFiles;
my %patches;
my %successors;
for my $manifest (glob "$manifestDir/*.nixmanifest") {
# print STDERR "reading $manifest\n";
readManifest $manifest, \%narFiles, \%patches, \%successors;
}
# Build a graph of all store paths that might contribute to the
# construction of $targetPath, and the special node "start". The
# edges are either patch operations, or downloads of full NAR files.
# The latter edges only occur between "start" and a store path.
my %graph;
$graph{"start"} = {d => 0, pred => undef, edges => []};
my @queue = ();
my $queueFront = 0;
my %done;
sub addToQueue {
my $v = shift;
return if defined $done{$v};
$done{$v} = 1;
push @queue, $v;
}
sub addNode {
my $u = shift;
$graph{$u} = {d => 999999999999, pred => undef, edges => []}
unless defined $graph{$u};
}
sub addEdge {
my $u = shift;
my $v = shift;
my $w = shift;
my $type = shift;
my $info = shift;
addNode $u;
push @{$graph{$u}->{edges}},
{weight => $w, start => $u, end => $v, type => $type, info => $info};
my $n = scalar @{$graph{$u}->{edges}};
}
addToQueue $targetPath;
sub isValidPath {
my $p = shift;
system "nix-store --isvalid '$p' 2> /dev/null";
return $? == 0;
}
while ($queueFront < scalar @queue) {
my $u = $queue[$queueFront++];
# print "$u\n";
addNode $u;
# If the path already exists, it has distance 0 from the "start"
# node.
if (isValidPath($u)) {
addEdge "start", $u, 0, "present", undef;
}
else {
# Add patch edges.
my $patchList = $patches{$u};
foreach my $patch (@{$patchList}) {
if (isValidPath($patch->{basePath})) {
# !!! this should be cached
my $hash = `nix-hash "$patch->{basePath}"`;
chomp $hash;
# print " MY HASH is $hash\n";
if ($hash ne $patch->{baseHash}) {
print LOGFILE "$$ rejecting $patch->{basePath}\n";
next;
}
}
# print " PATCH from $patch->{basePath}\n";
addToQueue $patch->{basePath};
addEdge $patch->{basePath}, $u, $patch->{size}, "patch", $patch;
}
# Add NAR file edges to the start node.
my $narFileList = $narFiles{$u};
foreach my $narFile (@{$narFileList}) {
# print " NAR from $narFile->{url}\n";
addEdge "start", $u, $narFile->{size}, "narfile", $narFile;
if ($u eq $targetPath) {
print LOGFILE "$$ full-download-would-be $narFile->{size}\n";
}
}
}
}
# Run Dijkstra's shortest path algorithm to determine the shortest
# sequence of download and/or patch actions that will produce
# $targetPath.
sub byDistance { # sort by distance, reversed
return -($graph{$a}->{d} <=> $graph{$b}->{d});
}
my @todo = keys %graph;
while (scalar @todo > 0) {
# Remove the closest element from the todo list.
@todo = sort byDistance @todo;
my $u = pop @todo;
my $u_ = $graph{$u};
# print "IN $u $u_->{d}\n";
foreach my $edge (@{$u_->{edges}}) {
my $v_ = $graph{$edge->{end}};
if ($v_->{d} > $u_->{d} + $edge->{weight}) {
$v_->{d} = $u_->{d} + $edge->{weight};
# Store the edge; to edge->start is actually the
# predecessor.
$v_->{pred} = $edge;
# print " RELAX $edge->{end} $v_->{d}\n";
}
}
}
# Retrieve the shortest path from "start" to $targetPath.
my @path = ();
my $cur = $targetPath;
die "don't know how to produce $targetPath\n"
unless defined $graph{$targetPath}->{pred};
while ($cur ne "start") {
push @path, $graph{$cur}->{pred};
$cur = $graph{$cur}->{pred}->{start};
}
# Traverse the shortest path, perform the actions described by the
# edges.
my $curStep = 1;
my $maxStep = scalar @path;
sub downloadFile {
my $url = shift;
my $hash = shift;
$ENV{"PRINT_PATH"} = 1;
$ENV{"QUIET"} = 1;
my ($hash2, $path) = `nix-prefetch-url '$url' '$hash'`;
chomp $hash2;
chomp $path;
die "hash mismatch" if $hash ne $hash2;
return $path;
}
while (scalar @path > 0) {
my $edge = pop @path;
my $u = $edge->{start};
my $v = $edge->{end};
print "\n*** Step $curStep/$maxStep: ";
$curStep++;
if ($edge->{type} eq "present") {
print "using already present path `$v'\n";
print LOGFILE "$$ present $v\n";
}
elsif ($edge->{type} eq "patch") {
my $patch = $edge->{info};
print "applying patch `$patch->{url}' to `$u' to create `$v'\n";
print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n";
# Download the patch.
print " downloading patch...\n";
my $patchPath = downloadFile "$patch->{url}", "$patch->{hash}";
# Turn the base path into a NAR archive, to which we can
# actually apply the patch.
print " packing base path...\n";
system "nix-store --dump $patch->{basePath} > /tmp/nar";
die "cannot dump `$patch->{basePath}'" if ($? != 0);
# Apply the patch.
print " applying patch...\n";
system "@libexecdir@/bspatch /tmp/nar /tmp/nar2 $patchPath";
die "cannot apply patch `$patchPath' to /tmp/nar" if ($? != 0);
# Unpack the resulting NAR archive into the target path.
print " unpacking patched archive...\n";
system "nix-store --restore $v < /tmp/nar2";
die "cannot unpack /tmp/nar2 into `$v'" if ($? != 0);
}
elsif ($edge->{type} eq "narfile") {
my $narFile = $edge->{info};
print "downloading `$narFile->{url}' into `$v'\n";
print LOGFILE "$$ narfile $narFile->{url} $narFile->{size} $v\n";
# Download the archive.
print " downloading archive...\n";
my $narFilePath = downloadFile "$narFile->{url}", "$narFile->{hash}";
# Unpack the archive into the target path.
print " unpacking archive...\n";
system "bunzip2 < '$narFilePath' | nix-store --restore '$v'";
die "cannot unpack `$narFilePath' into `$v'" if ($? != 0);
}
}
print LOGFILE "$$ success\n";
close LOGFILE;
-285
View File
@@ -1,285 +0,0 @@
#! /usr/bin/perl -w -I/home/eelco/nix/scripts
use strict;
use POSIX qw(tmpnam);
use readmanifest;
die unless scalar @ARGV == 5;
my $cacheDir = $ARGV[0];
my $patchesDir = $ARGV[1];
my $patchesURL = $ARGV[2];
my $srcDir = $ARGV[3];
my $dstDir = $ARGV[4];
my $tmpdir;
do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
print "TEMP = $tmpdir\n";
#END { rmdir $tmpdir; }
my %srcNarFiles;
my %srcPatches;
my %srcSuccessors;
my %dstNarFiles;
my %dstPatches;
my %dstSuccessors;
readManifest "$srcDir/MANIFEST",
\%srcNarFiles, \%srcPatches, \%srcSuccessors;
readManifest "$dstDir/MANIFEST",
\%dstNarFiles, \%dstPatches, \%dstSuccessors;
sub findOutputPaths {
my $narFiles = shift;
my $successors = shift;
my %outPaths;
foreach my $p (keys %{$narFiles}) {
# Ignore store expressions.
next if ($p =~ /\.store$/);
# Ignore builders (too much ambiguity -- they're all called
# `builder.sh').
next if ($p =~ /\.sh$/);
next if ($p =~ /\.patch$/);
# Don't bother including tar files etc.
next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/);
$outPaths{$p} = 1;
}
return %outPaths;
}
print "finding src output paths...\n";
my %srcOutPaths = findOutputPaths \%srcNarFiles, \%srcSuccessors;
print "finding dst output paths...\n";
my %dstOutPaths = findOutputPaths \%dstNarFiles, \%dstSuccessors;
sub getNameVersion {
my $p = shift;
$p =~ /\/[0-9a-f]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
my $name = $1;
my $version = $2;
$name =~ s/^-//;
$version =~ s/^-//;
return ($name, $version);
}
# A quick hack to get a measure of the `distance' between two
# versions: it's just the position of the first character that differs
# (or 999 if they are the same).
sub versionDiff {
my $s = shift;
my $t = shift;
my $i;
return 999 if $s eq $t;
for ($i = 0; $i < length $s; $i++) {
return $i if $i >= length $t or
substr($s, $i, 1) ne substr($t, $i, 1);
}
return $i;
}
sub getNarBz2 {
my $narFiles = shift;
my $storePath = shift;
my $narFileList = $$narFiles{$storePath};
die "missing store expression $storePath" unless defined $narFileList;
my $narFile = @{$narFileList}[0];
die unless defined $narFile;
$narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1;
return "$cacheDir/$1";
}
sub containsPatch {
my $patches = shift;
my $storePath = shift;
my $basePath = shift;
my $patchList = $$patches{$storePath};
return 0 if !defined $patchList;
my $found = 0;
foreach my $patch (@{$patchList}) {
# !!! baseHash might differ
return 1 if $patch->{basePath} eq $basePath;
}
return 0;
}
# For each output path in the destination, see if we need to / can
# create a patch.
print "creating patches...\n";
foreach my $p (keys %dstOutPaths) {
# If exactly the same path already exists in the source, skip it.
next if defined $srcOutPaths{$p};
# print " $p\n";
# If not, then we should find the path in the source that is
# `most' likely to be present on a system that wants to install
# this path.
(my $name, my $version) = getNameVersion $p;
my @closest = ();
my $closestVersion;
my $minDist = -1; # actually, larger means closer
# Find all source paths with the same name.
foreach my $q (keys %srcOutPaths) {
(my $name2, my $version2) = getNameVersion $q;
if ($name eq $name2) {
my $dist = versionDiff $version, $version2;
if ($dist > $minDist) {
$minDist = $dist;
@closest = ($q);
$closestVersion = $version2;
} elsif ($dist == $minDist) {
push @closest, $q;
}
}
}
if (scalar(@closest) == 0) {
print " NO BASE: $p\n";
next;
}
foreach my $closest (@closest) {
# Generate a patch between $closest and $p.
print " $p <- $closest\n";
# If the patch already exists, skip it.
if (containsPatch(\%srcPatches, $p, $closest) ||
containsPatch(\%dstPatches, $p, $closest))
{
print " skipping, already exists\n";
next;
}
# next;
my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
system("bunzip2 < $srcNarBz2 > $tmpdir/A") == 0
or die "cannot unpack $srcNarBz2";
system("bunzip2 < $dstNarBz2 > $tmpdir/B") == 0
or die "cannot unpack $dstNarBz2";
system("bsdiff $tmpdir/A $tmpdir/B $tmpdir/DIFF") == 0
or die "cannot compute binary diff";
my $baseHash = `nix-hash --flat $tmpdir/A` or die;
chomp $baseHash;
my $narHash = `nix-hash --flat $tmpdir/B` or die;
chomp $narHash;
my $narDiffHash = `nix-hash --flat $tmpdir/DIFF` or die;
chomp $narDiffHash;
my $narDiffSize = (stat "$tmpdir/DIFF")[7];
my $dstNarBz2Size = (stat $dstNarBz2)[7];
if ($narDiffSize >= $dstNarBz2Size) {
print " rejecting; patch bigger than full archive\n";
next;
}
my $finalName =
"$narDiffHash-$name-$closestVersion-to-$version.nar-bsdiff";
print " size $narDiffSize; full size $dstNarBz2Size\n";
if (-e "$patchesDir/$finalName") {
print " not copying, already exists\n";
}
else {
system("cp '$tmpdir/DIFF' '$patchesDir/$finalName.tmp'") == 0
or die "cannot copy diff";
rename("$patchesDir/$finalName.tmp", "$patchesDir/$finalName")
or die "cannot rename $patchesDir/$finalName.tmp";
}
# Add the patch to the manifest.
addPatch \%dstPatches, $p,
{ url => "$patchesURL/$finalName", hash => $narDiffHash
, size => $narDiffSize
, basePath => $closest, baseHash => $baseHash
, narHash => $narHash, patchType => "nar-bsdiff"
};
}
}
# Add in any potentially useful patches in the source (namely, those
# patches that produce either paths in the destination or paths that
# can be used as the base for other useful patches).
my $changed;
do {
# !!! we repeat this to reach the transitive closure; inefficient
$changed = 0;
foreach my $p (keys %srcPatches) {
my $patchList = $srcPatches{$p};
my $include = 0;
# Is path $p included in the destination? If so, include
# patches that produce it.
$include = 1 if (defined $dstNarFiles{$p});
# Is path $p a path that serves as a base for paths in the
# destination? If so, include patches that produce it.
foreach my $q (keys %dstPatches) {
foreach my $patch (@{$dstPatches{$q}}) {
# !!! check baseHash
$include = 1 if ($p eq $patch->{basePath});
}
}
if ($include) {
foreach my $patch (@{$patchList}) {
$changed = 1 if addPatch \%dstPatches, $p, $patch;
}
}
}
} while $changed;
# Rewrite the manifest of the destination (with the new patches).
writeManifest "$dstDir/MANIFEST",
\%dstNarFiles, \%dstPatches, \%dstSuccessors;
Regular → Executable
View File
File mode changed.
Regular → Executable
-4
View File
@@ -53,10 +53,6 @@ sub addChannel {
sub update {
readChannels;
# Get rid of all the old substitutes.
system "@bindir@/nix-store --clear-substitutes";
die "cannot clear substitutes" if ($? != 0);
# Pull cache manifests.
foreach my $url (@channels) {
print "pulling cache manifest from `$url'\n";
Regular → Executable
View File
File mode changed.
+21 -29
View File
@@ -1,41 +1,35 @@
#! @shell@ -e
url=$1
hash=$2
if test -z "$url"; then
echo "syntax: nix-prefetch-url URL" >&2
exit 1
fi
# Determine the hash, unless it was given.
if test -z "$hash"; then
# !!! race
tmpPath1=@storedir@/nix-prefetch-url-$$
# !!! race
tmpPath1=@storedir@/nix-prefetch-url-$$
# Test whether we have write permission in the store. If not, fetch
# to /tmp and don't copy to the store. This is a hack to make this
# script at least work somewhat in setuid installations.
if ! touch $tmpPath1 2> /dev/null; then
echo "(cannot write to the store, result won't be cached)" >&2
dummyMode=1
tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
fi
# Test whether we have write permission in the store. If not,
# fetch to /tmp and don't copy to the store. This is a hack to
# make this script at least work somewhat in setuid installations.
if ! touch $tmpPath1 2> /dev/null; then
echo "(cannot write to the store, result won't be cached)" >&2
dummyMode=1
tmpPath1=/tmp/nix-prefetch-url-$$ # !!! security?
fi
# Perform the checkout.
@curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
# Perform the checkout.
@curl@ --fail --location --max-redirs 20 "$url" > $tmpPath1
# Compute the hash.
hash=$(@bindir@/nix-hash --flat $tmpPath1)
if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
# Rename it so that the fetchurl builder can find it.
if test "$dummyMode" != 1; then
tmpPath2=@storedir@/nix-prefetch-url-$hash
test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
fi
# Compute the hash.
hash=$(@bindir@/nix-hash --flat $tmpPath1)
echo "hash is $hash" >&2
# Rename it so that the fetchurl builder can find it.
if test "$dummyMode" != 1; then
tmpPath2=@storedir@/nix-prefetch-url-$hash
test -e $tmpPath2 || mv $tmpPath1 $tmpPath2 # !!! race
fi
# Create a Nix expression that does a fetchurl.
@@ -47,11 +41,9 @@ storeExpr=$( \
# Realise it.
finalPath=$(@bindir@/nix-store -qnB --force-realise $storeExpr)
if ! test -n "$QUIET"; then echo "path is $finalPath" >&2; fi
echo "path is $finalPath" >&2
if test -n "$tmpPath1" -o -n "$tmpPath2"; then
rm -rf $tmpPath1 $tmpPath2 || true
fi
rm -rf $tmpPath1 $tmpPath2 || true
echo $hash
+41 -40
View File
@@ -10,52 +10,53 @@ do { $tmpdir = tmpnam(); }
until mkdir $tmpdir, 0777;
my $manifest = "$tmpdir/manifest";
my $confFile = "@sysconfdir@/nix/prebuilts.conf";
END { unlink $manifest; rmdir $tmpdir; }
#END { unlink $manifest; rmdir $tmpdir; }
# Obtain URLs either from the command line or from a configuration file.
my %narFiles;
my %patches;
my %storePaths2urls;
my %urls2hashes;
my %successors;
sub processURL {
sub doURL {
my $url = shift;
$url =~ s/\/$//;
print "obtaining list of Nix archives at $url...\n";
system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
"'$url' > '$manifest'") == 0
or die "curl failed: $?";
readManifest $manifest, \%narFiles, \%patches, \%successors;
my $baseName = "unnamed";
if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component
$baseName = $1;
processURL $manifest, $url, \%storePaths2urls, \%urls2hashes, \%successors;
}
if (scalar @ARGV > 0) {
while (@ARGV) {
my $url = shift @ARGV;
doURL $url;
}
my $hash = `@bindir@/nix-hash --flat '$manifest'`
or die "cannot hash `$manifest'";
chomp $hash;
my $finalPath = "@localstatedir@/nix/manifests/$baseName-$hash.nixmanifest";
system("mv '$manifest' '$finalPath'") == 0
or die "cannot move `$manifest' to `$finalPath";
}
while (@ARGV) {
my $url = shift @ARGV;
processURL $url;
} else {
open CONFFILE, "<$confFile";
while (<CONFFILE>) {
chomp;
if (/^\s*(\S+)\s*(\#.*)?$/) {
my $url = $1;
doURL $url;
}
}
close CONFFILE;
}
my $size = scalar (keys %narFiles);
my $size = scalar (keys %storePaths2urls);
print "$size store paths in manifest\n";
# Instantiate a store expression that builds the substitute program
# (the program that fetches URLs and unpacks them into the store).
my $nixExpr =
"(import @datadir@/nix/corepkgs/nix-pull) " .
"{system = \"@system@\";}";
print STDERR "instantiating store expression...\n";
my $storeExpr = `echo '$nixExpr' | @bindir@/nix-instantiate -`
or die "cannot instantiate Nix expression";
chomp $storeExpr;
# Register all substitutes.
print STDERR "registering substitutes...\n";
@@ -64,13 +65,13 @@ my $pid = open2(\*READ, \*WRITE, "@bindir@/nix-store --substitute")
close READ;
foreach my $storePath (keys %narFiles) {
my $narFileList = $narFiles{$storePath};
foreach my $narFile (@{$narFileList}) {
print WRITE "$storePath\n";
print WRITE "@libexecdir@/nix/download-using-manifests.pl\n";
print WRITE "0\n";
}
foreach my $storePath (keys %storePaths2urls) {
print WRITE "$storePath\n";
print WRITE "$storeExpr\n";
print WRITE "/fetch\n";
print WRITE "2\n";
print WRITE "$storePaths2urls{$storePath}\n";
print WRITE "$urls2hashes{$storePaths2urls{$storePath}}\n";
}
close WRITE;
+26 -39
View File
@@ -2,7 +2,6 @@
use strict;
use POSIX qw(tmpnam);
use readmanifest;
my $tmpdir;
do { $tmpdir = tmpnam(); }
@@ -26,7 +25,7 @@ my $manifest_put_url = shift @ARGV;
# From the given store expressions, determine the requisite store
# paths.
my %storePaths;
my %storepaths;
foreach my $storeexpr (@ARGV) {
die unless $storeexpr =~ /^\//;
@@ -40,12 +39,12 @@ foreach my $storeexpr (@ARGV) {
while (<PATHS>) {
chomp;
die "bad: $_" unless /^\//;
$storePaths{$_} = "";
$storepaths{$_} = "";
}
close PATHS;
}
my @storePaths = keys %storePaths;
my @storepaths = keys %storepaths;
# For each path, create a Nix expression that turns the path into
@@ -53,14 +52,14 @@ my @storePaths = keys %storePaths;
open NIX, ">$nixfile";
print NIX "[";
foreach my $storePath (@storePaths) {
die unless ($storePath =~ /\/[0-9a-z]{32}.*$/);
foreach my $storepath (@storepaths) {
die unless ($storepath =~ /\/[0-9a-z]{32}.*$/);
# Construct a Nix expression that creates a Nix archive.
my $nixexpr =
"((import @datadir@/nix/corepkgs/nar/nar.nix) " .
# !!! $storePath should be represented as a closure
"{path = \"$storePath\"; system = \"@system@\";}) ";
# !!! $storepath should be represented as a closure
"{path = \"$storepath\"; system = \"@system@\";}) ";
print NIX $nixexpr;
}
@@ -109,65 +108,53 @@ while (scalar @tmp > 0) {
# Create the manifest.
print STDERR "creating manifest...\n";
my %narFiles;
my %patches;
my %successors;
open MANIFEST, ">$manifest";
my @nararchives;
for (my $n = 0; $n < scalar @storePaths; $n++) {
my $storePath = $storePaths[$n];
for (my $n = 0; $n < scalar @storepaths; $n++) {
my $storepath = $storepaths[$n];
my $nardir = $narpaths[$n];
$storePath =~ /\/([^\/]*)$/;
$storepath =~ /\/([^\/]*)$/;
my $basename = $1;
defined $basename or die;
my $narname = "$basename.nar.bz2";
my $narfile = "$nardir/$narname";
(-f $narfile) or die "narfile for $storePath not found";
(-f $narfile) or die "narfile for $storepath not found";
push @nararchives, $narfile;
open MD5, "$nardir/narbz2-hash" or die "cannot open narbz2-hash";
my $narbz2Hash = <MD5>;
chomp $narbz2Hash;
$narbz2Hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
open MD5, "$nardir/md5" or die "cannot open hash";
my $hash = <MD5>;
chomp $hash;
$hash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
close MD5;
open MD5, "$nardir/nar-hash" or die "cannot open nar-hash";
my $narHash = <MD5>;
chomp $narHash;
$narHash =~ /^[0-9a-z]{32}$/ or die "invalid hash";
close MD5;
my $narbz2Size = (stat $narfile)[7];
print MANIFEST "{\n";
print MANIFEST " StorePath: $storepath\n";
print MANIFEST " NarURL: $archives_get_url/$narname\n";
print MANIFEST " MD5: $hash\n";
$narFiles{$storePath} = [
{ url => $archives_get_url/$narname
, hash => $narbz2Hash
, size => $narbz2Size
, narHash => $narHash
}
];
if ($storePath =~ /\.store$/) {
open PREDS, "@bindir@/nix-store --query --predecessors $storePath |" or die "cannot run nix";
if ($storepath =~ /\.store$/) {
open PREDS, "@bindir@/nix-store --query --predecessors $storepath |" or die "cannot run nix";
while (<PREDS>) {
chomp;
die unless (/^\//);
my $pred = $_;
# Only include predecessors that are themselves being
# pushed.
if (defined $storePaths{$pred}) {
$successors{$pred} = $storePath;
if (defined $storepaths{$pred}) {
print MANIFEST " SuccOf: $pred\n";
}
}
close PREDS;
}
print MANIFEST "}\n";
}
writeManifest $manifest, \%narFiles, \%patches, \%successors;
close MANIFEST;
# Upload the archives.
-69
View File
@@ -1,69 +0,0 @@
#! /usr/bin/perl -w
use strict;
{ my $ofh = select STDOUT;
$| = 1;
select $ofh;
}
#my @paths = ("/nix/store/caef3a49150506d233f474322a824e50-glibc-2.3.3", "/nix/store/a8a9d585d1ad4b1bc911be7743b3b996-glibc-2.3.3");
my @paths = ("/nix/store");
my $tmpfile = "/tmp/nix-optimise-hash-list";
#my $tmpfile = "/data/nix-optimise-hash-list";
system("find @paths -type f -print0 | xargs -0 md5sum -- > $tmpfile") == 0
or die "cannot hash store files";
system("sort $tmpfile > $tmpfile.sorted") == 0
or die "cannot sort list";
open LIST, "<$tmpfile.sorted" or die;
my $prevFile;
my $prevHash;
my $totalSpace = 0;
my $savedSpace = 0;
my $files = 0;
while (<LIST>) {
# print "D";
/^([0-9a-f]*)\s+(.*)$/ or die;
my $curFile = $2;
my $curHash = $1;
# print "A";
my $fileSize = (stat $curFile)[7];
# print "B";
# my $fileSize = 1;
$totalSpace += $fileSize;
if (defined $prevHash && $curHash eq $prevHash) {
# print "$curFile = $prevFile\n";
$savedSpace += $fileSize;
} else {
$prevFile = $curFile;
$prevHash = $curHash;
}
print "." if ($files++ % 100 == 0);
#print ".";
# print "C";
}
print "\n";
print "total space = $totalSpace\n";
print "saved space = $savedSpace\n";
my $savings = ($savedSpace / $totalSpace) * 100.0;
print "savings = $savings %\n";
close LIST;
+32 -153
View File
@@ -1,54 +1,27 @@
use strict;
sub addPatch {
my $patches = shift;
my $storePath = shift;
my $patch = shift;
$$patches{$storePath} = []
unless defined $$patches{$storePath};
my $patchList = $$patches{$storePath};
my $found = 0;
foreach my $patch2 (@{$patchList}) {
if ($patch2->{url} eq $patch->{url}) {
if ($patch2->{hash} eq $patch->{hash}) {
$found = 1 if ($patch2->{basePath} eq $patch->{basePath});
} else {
die "conflicting hashes for URL $patch->{url}, " .
"namely $patch2->{hash} and $patch->{hash}";
}
}
}
push @{$patchList}, $patch if !$found;
return !$found;
}
sub readManifest {
sub processURL {
my $manifest = shift;
my $narFiles = shift;
my $patches = shift;
my $url = shift;
my $storePaths2urls = shift;
my $urls2hashes = shift;
my $successors = shift;
$url =~ s/\/$//;
print "obtaining list of Nix archives at $url...\n";
system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
"'$url' > '$manifest'") == 0
or die "curl failed: $?";
open MANIFEST, "<$manifest";
my $inside = 0;
my $type;
my $storePath;
my $url;
my $narurl;
my $hash;
my $size;
my @preds;
my $basePath;
my $baseHash;
my $patchType;
my $narHash;
while (<MANIFEST>) {
chomp;
@@ -56,82 +29,39 @@ sub readManifest {
next if (/^$/);
if (!$inside) {
if (/^\s*(\w*)\s*\{$/) {
$type = $1;
$type = "narfile" if $type eq "";
if (/^\{$/) {
$inside = 1;
undef $storePath;
undef $url;
undef $narurl;
undef $hash;
undef $size;
@preds = ();
undef $narHash;
undef $basePath;
undef $baseHash;
undef $patchType;
}
else { die "bad line: $_"; }
} else {
if (/^\}$/) {
$inside = 0;
if ($type eq "narfile") {
$$storePaths2urls{$storePath} = $narurl;
$$urls2hashes{$narurl} = $hash;
$$narFiles{$storePath} = []
unless defined $$narFiles{$storePath};
my $narFileList = $$narFiles{$storePath};
my $found = 0;
foreach my $narFile (@{$narFileList}) {
if ($narFile->{url} eq $url) {
if ($narFile->{hash} eq $hash) {
$found = 1;
} else {
die "conflicting hashes for URL $url, " .
"namely $narFile->{hash} and $hash";
}
}
}
if (!$found) {
push @{$narFileList},
{ url => $url, hash => $hash, size => $size
, narHash => $narHash
};
}
foreach my $p (@preds) {
$$successors{$p} = $storePath;
}
}
elsif ($type eq "patch") {
addPatch $patches, $storePath,
{ url => $url, hash => $hash, size => $size
, basePath => $basePath, baseHash => $baseHash
, narHash => $narHash, patchType => $patchType
};
foreach my $p (@preds) {
$$successors{$p} = $storePath;
}
}
elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; }
elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; }
elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; }
elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; }
elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) { push @preds, $1; }
elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; }
elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; }
elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; }
elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; }
# Compatibility;
elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; }
elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = $1; }
elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) {
$storePath = $1;
}
elsif (/^\s*NarURL:\s*(\S+)\s*$/) {
$narurl = $1;
}
elsif (/^\s*MD5:\s*(\S+)\s*$/) {
$hash = $1;
}
elsif (/^\s*SuccOf:\s*(\/\S+)\s*$/) {
push @preds, $1;
}
else { die "bad line: $_"; }
}
}
@@ -139,55 +69,4 @@ sub readManifest {
}
sub writeManifest
{
my $manifest = shift;
my $narFiles = shift;
my $patches = shift;
my $successors = shift;
open MANIFEST, ">$manifest.tmp"; # !!! check exclusive
foreach my $storePath (keys %{$narFiles}) {
my $narFileList = $$narFiles{$storePath};
foreach my $narFile (@{$narFileList}) {
print MANIFEST "{\n";
print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $narFile->{url}\n";
print MANIFEST " MD5: $narFile->{hash}\n";
print MANIFEST " NarHash: $narFile->{narHash}\n";
print MANIFEST " Size: $narFile->{size}\n";
foreach my $p (keys %{$successors}) { # !!! quadratic
if ($$successors{$p} eq $storePath) {
print MANIFEST " SuccOf: $p\n";
}
}
print MANIFEST "}\n";
}
}
foreach my $storePath (keys %{$patches}) {
my $patchList = $$patches{$storePath};
foreach my $patch (@{$patchList}) {
print MANIFEST "patch {\n";
print MANIFEST " StorePath: $storePath\n";
print MANIFEST " NarURL: $patch->{url}\n";
print MANIFEST " MD5: $patch->{hash}\n";
print MANIFEST " NarHash: $patch->{narHash}\n";
print MANIFEST " Size: $patch->{size}\n";
print MANIFEST " BasePath: $patch->{basePath}\n";
print MANIFEST " BaseHash: $patch->{baseHash}\n";
print MANIFEST " Type: $patch->{patchType}\n";
print MANIFEST "}\n";
}
}
close MANIFEST;
rename("$manifest.tmp", $manifest)
or die "cannot rename $manifest.tmp: $!";
}
return 1;
-19
View File
@@ -1,19 +0,0 @@
#! /usr/bin/perl -w -I/home/eelco/nix/scripts
use strict;
use readmanifest;
for my $p (@ARGV) {
my %narFiles;
my %patches;
my %successors;
readManifest $p,
\%narFiles, \%patches, \%successors;
%patches = ();
writeManifest $p,
\%narFiles, \%patches, \%successors;
}
-53
View File
@@ -1,53 +0,0 @@
#! /usr/bin/perl -w -I.
use strict;
use readmanifest;
die unless scalar @ARGV == 2;
my $cache = $ARGV[0];
my $manifest = $ARGV[1];
my %narFiles;
my %patches;
my %successors;
readManifest $manifest, \%narFiles, \%patches, \%successors;
foreach my $storePath (keys %narFiles) {
my $narFileList = $narFiles{$storePath};
foreach my $narFile (@{$narFileList}) {
if (!defined $narFile->{size} or
!defined $narFile->{narHash})
{
$narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1;
my $fn = "$cache/$1";
my @info = stat $fn or die;
$narFile->{size} = $info[7];
my $narHash;
my $hashFile = "$fn.NARHASH";
if (-e $hashFile) {
open HASH, "<$hashFile" or die;
$narHash = <HASH>;
close HASH;
} else {
print "$fn\n";
$narHash = `bunzip2 < '$fn' | nix-hash --flat /dev/stdin` or die;
open HASH, ">$hashFile" or die;
print HASH $narHash;
close HASH;
}
chomp $narHash;
$narFile->{narHash} = $narHash;
}
}
}
if (! -e "$manifest.backup") {
system "mv --reply=no '$manifest' '$manifest.backup'";
}
writeManifest $manifest, \%narFiles, \%patches, \%successors;
+1 -1
View File
@@ -1,5 +1,5 @@
SUBDIRS = bin2c boost libutil libstore libmain nix-store nix-hash \
libexpr nix-instantiate nix-env log2xml bsdiff-4.2
libexpr nix-instantiate nix-env log2xml
EXTRA_DIST = aterm-helper.pl
-121
View File
@@ -1,121 +0,0 @@
BSD Protection License
February 2002
Preamble
--------
The Berkeley Software Distribution ("BSD") license has proven very effective
over the years at allowing for a wide spread of work throughout both
commercial and non-commercial products. For programmers whose primary
intention is to improve the general quality of available software, it is
arguable that there is no better license than the BSD license, as it
permits improvements to be used wherever they will help, without idealogical
or metallic constraint.
This is of particular value to those who produce reference implementations
of proposed standards: The case of TCP/IP clearly illustrates that freely
and universally available implementations leads the rapid acceptance of
standards -- often even being used instead of a de jure standard (eg, OSI
network models).
With the rapid proliferation of software licensed under the GNU General
Public License, however, the continued success of this role is called into
question. Given that the inclusion of a few lines of "GPL-tainted" work
into a larger body of work will result in restricted distribution -- and
given that further work will likely build upon the "tainted" portions,
making them difficult to remove at a future date -- there are inevitable
circumstances where authors would, in order to protect their goal of
providing for the widespread usage of their work, wish to guard against
such "GPL-taint".
In addition, one can imagine that companies which operate by producing and
selling (possibly closed-source) code would wish to protect themselves
against the rise of a GPL-licensed competitor. While under existing
licenses this would mean not releasing their code under any form of open
license, if a license existed under which they could incorporate any
improvements back into their own (commercial) products then they might be
far more willing to provide for non-closed distribution.
For the above reasons, we put forth this "BSD Protection License": A
license designed to retain the freedom granted by the BSD license to use
licensed works in a wide variety of settings, both non-commercial and
commercial, while protecting the work from having future contributors
restrict that freedom.
The precise terms and conditions for copying, distribution, and
modification follow.
BSD PROTECTION LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION, AND MODIFICATION
----------------------------------------------------------------
0. Definitions.
a) "Program", below, refers to any program or work distributed under
the terms of this license.
b) A "work based on the Program", below, refers to either the Program
or any derivative work under copyright law.
c) "Modification", below, refers to the act of creating derivative works.
d) "You", below, refers to each licensee.
1. Scope.
This license governs the copying, distribution, and modification of the
Program. Other activities are outside the scope of this license; The
act of running the Program is not restricted, and the output from the
Program is covered only if its contents constitute a work based on the
Program.
2. Verbatim copies.
You may copy and distribute verbatim copies of the Program as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice; keep
intact all the notices that refer to this License and to the absence of
any warranty; and give any other recipients of the Program a copy of this
License along with the Program.
3. Modification and redistribution under closed license.
You may modify your copy or copies of the Program, and distribute
the resulting derivative works, provided that you meet the
following conditions:
a) The copyright notice and disclaimer on the Program must be reproduced
and included in the source code, documentation, and/or other materials
provided in a manner in which such notices are normally distributed.
b) The derivative work must be clearly identified as such, in order that
it may not be confused with the original work.
c) The license under which the derivative work is distributed must
expressly prohibit the distribution of further derivative works.
4. Modification and redistribution under open license.
You may modify your copy or copies of the Program, and distribute
the resulting derivative works, provided that you meet the
following conditions:
a) The copyright notice and disclaimer on the Program must be reproduced
and included in the source code, documentation, and/or other materials
provided in a manner in which such notices are normally distributed.
b) You must clearly indicate the nature and date of any changes made
to the Program. The full details need not necessarily be included in
the individual modified files, provided that each modified file is
clearly marked as such and instructions are included on where the
full details of the modifications may be found.
c) You must cause any work that you distribute or publish, that in whole
or in part contains or is derived from the Program or any part
thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
5. Implied acceptance.
You may not copy or distribute the Program or any derivative works except
as expressly provided under this license. Consequently, any such action
will be taken as implied acceptance of the terms of this license.
6. NO WARRANTY.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
THE COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE FOR ANY DIRECT,
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
TORT, EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
-7
View File
@@ -1,7 +0,0 @@
libexec_PROGRAMS = bsdiff bspatch
bsdiff_SOURCES = bsdiff.c
bspatch_SOURCES = bspatch.c
AM_CFLAGS = -O3 -DBZIP2=\"$(bzip2)\"
-18
View File
@@ -1,18 +0,0 @@
CFLAGS += -O3
.ifdef BZIP2
CFLAGS += -DBZIP2=\"${BZIP2}\"
.endif
PREFIX ?= /usr/local
INSTALL_PROGRAM ?= ${INSTALL} -c -s -m 555
INSTALL_MAN ?= ${INSTALL} -c -m 444
all: bsdiff bspatch
bsdiff: bsdiff.c
bspatch: bspatch.c
install:
${INSTALL_PROGRAM} bsdiff bspatch ${PREFIX}/bin
.ifndef WITHOUT_MAN
${INSTALL_MAN} bsdiff.1 bspatch.1 ${PREFIX}/man/man1
.endif
-36
View File
@@ -1,36 +0,0 @@
.Dd May 18, 2003
.Dt BSDIFF 1
.Os FreeBSD
.Sh NAME
.Nm bsdiff
.Nd generate a patch between two binary files
.Sh SYNOPSIS
.Nm
.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
.Sh DESCRIPTION
.Nm
compares
.Ao Ar oldfile Ac
to
.Ao Ar newfile Ac
and writes to
.Ao Ar patchfile Ac
a binary patch suitable for use by bspatch(1).
When
.Ao Ar oldfile Ac
and
.Ao Ar newfile Ac
are two versions of an executable program, the
patches produced are on average a factor of five smaller
than those produced by any other binary patch tool known
to the author.
.Pp
.Nm
uses memory equal to 17 times the size of
.Ao Ar oldfile Ac ,
and requires
an absolute minimum working set size of 8 times the size of oldfile.
.Sh SEE ALSO
.Xr bspatch 1
.Sh AUTHORS
.An Colin Percival Aq cperciva@daemonology.net
-385
View File
@@ -1,385 +0,0 @@
/*
bsdiff.c -- Binary patch generator.
Copyright 2003 Colin Percival
For the terms under which this work may be distributed, please see
the adjoining file "LICENSE".
*/
#ifndef BZIP2
#define BZIP2 "/usr/bin/bzip2"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <err.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/wait.h>
#define MIN(x,y) (((x)<(y)) ? (x) : (y))
void split(off_t *I,off_t *V,off_t start,off_t len,off_t h)
{
off_t i,j,k,x,tmp,jj,kk;
if(len<16) {
for(k=start;k<start+len;k+=j) {
j=1;x=V[I[k]+h];
for(i=1;k+i<start+len;i++) {
if(V[I[k+i]+h]<x) {
x=V[I[k+i]+h];
j=0;
};
if(V[I[k+i]+h]==x) {
tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp;
j++;
};
};
for(i=0;i<j;i++) V[I[k+i]]=k+j-1;
if(j==1) I[k]=-1;
};
return;
};
x=V[I[start+len/2]+h];
jj=0;kk=0;
for(i=start;i<start+len;i++) {
if(V[I[i]+h]<x) jj++;
if(V[I[i]+h]==x) kk++;
};
jj+=start;kk+=jj;
i=start;j=0;k=0;
while(i<jj) {
if(V[I[i]+h]<x) {
i++;
} else if(V[I[i]+h]==x) {
tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp;
j++;
} else {
tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp;
k++;
};
};
while(jj+j<kk) {
if(V[I[jj+j]+h]==x) {
j++;
} else {
tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp;
k++;
};
};
if(jj>start) split(I,V,start,jj-start,h);
for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1;
if(jj==kk-1) I[jj]=-1;
if(start+len>kk) split(I,V,kk,start+len-kk,h);
}
void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize)
{
off_t buckets[256];
off_t i,h,len;
for(i=0;i<256;i++) buckets[i]=0;
for(i=0;i<oldsize;i++) buckets[old[i]]++;
for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
for(i=255;i>0;i--) buckets[i]=buckets[i-1];
buckets[0]=0;
for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i;
I[0]=oldsize;
for(i=0;i<oldsize;i++) V[i]=buckets[old[i]];
V[oldsize]=0;
for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
I[0]=-1;
for(h=1;I[0]!=-(oldsize+1);h+=h) {
len=0;
for(i=0;i<oldsize+1;) {
if(I[i]<0) {
len-=I[i];
i-=I[i];
} else {
if(len) I[i-len]=-len;
len=V[I[i]]+1-i;
split(I,V,i,len,h);
i+=len;
len=0;
};
};
if(len) I[i-len]=-len;
};
for(i=0;i<oldsize+1;i++) I[V[i]]=i;
}
off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize)
{
off_t i;
for(i=0;(i<oldsize)&&(i<newsize);i++)
if(old[i]!=new[i]) break;
return i;
}
off_t search(off_t *I,u_char *old,off_t oldsize,
u_char *new,off_t newsize,off_t st,off_t en,off_t *pos)
{
off_t x,y;
if(en-st<2) {
x=matchlen(old+I[st],oldsize-I[st],new,newsize);
y=matchlen(old+I[en],oldsize-I[en],new,newsize);
if(x>y) {
*pos=I[st];
return x;
} else {
*pos=I[en];
return y;
}
};
x=st+(en-st)/2;
if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) {
return search(I,old,oldsize,new,newsize,x,en,pos);
} else {
return search(I,old,oldsize,new,newsize,st,x,pos);
};
}
void offtout(off_t x,u_char *buf)
{
off_t y;
if(x<0) y=-x; else y=x;
buf[0]=y%256;y-=buf[0];
y=y/256;buf[1]=y%256;y-=buf[1];
y=y/256;buf[2]=y%256;y-=buf[2];
y=y/256;buf[3]=y%256;y-=buf[3];
y=y/256;buf[4]=y%256;y-=buf[4];
y=y/256;buf[5]=y%256;y-=buf[5];
y=y/256;buf[6]=y%256;y-=buf[6];
y=y/256;buf[7]=y%256;
if(x<0) buf[7]|=0x80;
}
int main(int argc,char *argv[])
{
int fd,p[2];
pid_t pid;
u_char *old,*new;
off_t oldsize,newsize;
off_t *I,*V;
off_t scan,pos,len;
off_t lastscan,lastpos,lastoffset;
off_t oldscore,scsc;
off_t s,Sf,lenf,Sb,lenb;
off_t overlap,Ss,lens;
off_t i;
off_t dblen,eblen;
u_char *db,*eb;
u_char buf[8];
u_char header[32];
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
/* Allocate oldsize+1 bytes instead of oldsize bytes to ensure
that we never try to malloc(0) and get a NULL pointer */
if(((fd=open(argv[1],O_RDONLY,0))<0) ||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
((old=malloc(oldsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,old,oldsize)!=oldsize) ||
(close(fd)==-1)) err(1,"%s",argv[1]);
if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) ||
((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL);
qsufsort(I,V,old,oldsize);
free(V);
/* Allocate newsize+1 bytes instead of newsize bytes to ensure
that we never try to malloc(0) and get a NULL pointer */
if(((fd=open(argv[2],O_RDONLY,0))<0) ||
((newsize=lseek(fd,0,SEEK_END))==-1) ||
((new=malloc(newsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,new,newsize)!=newsize) ||
(close(fd)==-1)) err(1,"%s",argv[2]);
if(((db=malloc(newsize+1))==NULL) ||
((eb=malloc(newsize+1))==NULL)) err(1,NULL);
dblen=0;
eblen=0;
if((fd=open(argv[3],O_CREAT|O_TRUNC|O_WRONLY,0666))<0)
err(1,"%s",argv[3]);
/* Header is
0 8 "BSDIFF40"
8 8 length of bzip2ed ctrl block
16 8 length of bzip2ed diff block
24 8 length of new file */
/* File is
0 32 Header
32 ?? Bzip2ed ctrl block
?? ?? Bzip2ed diff block
?? ?? Bzip2ed extra block */
memcpy(header,"BSDIFF40",8);
memset(header+8,0,24);
if(write(fd,header,32)!=32) err(1,"%s",argv[3]);
if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
if(pid==0) {
if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
(dup2(p[0],0)==-1) || (close(fd)==-1) ||
(close(p[0])==-1) || (close(p[1])==-1))
err(1,NULL);
execl(BZIP2,BZIP2,"-zc",NULL);
err(1,"%s",BZIP2);
};
if(close(p[0])==-1) err(1,NULL);
scan=0;len=0;
lastscan=0;lastpos=0;lastoffset=0;
while(scan<newsize) {
oldscore=0;
for(scsc=scan+=len;scan<newsize;scan++) {
len=search(I,old,oldsize,new+scan,newsize-scan,
0,oldsize,&pos);
for(;scsc<scan+len;scsc++)
if((scsc+lastoffset<oldsize) &&
(old[scsc+lastoffset] == new[scsc]))
oldscore++;
if(((len==oldscore) && (len!=0)) ||
(len>oldscore+8)) break;
if((scan+lastoffset<oldsize) &&
(old[scan+lastoffset] == new[scan]))
oldscore--;
};
if((len!=oldscore) || (scan==newsize)) {
s=0;Sf=0;lenf=0;
for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
if(old[lastpos+i]==new[lastscan+i]) s++;
i++;
if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
};
lenb=0;
if(scan<newsize) {
s=0;Sb=0;
for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
if(old[pos-i]==new[scan-i]) s++;
if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
};
};
if(lastscan+lenf>scan-lenb) {
overlap=(lastscan+lenf)-(scan-lenb);
s=0;Ss=0;lens=0;
for(i=0;i<overlap;i++) {
if(new[lastscan+lenf-overlap+i]==
old[lastpos+lenf-overlap+i]) s++;
if(new[scan-lenb+i]==
old[pos-lenb+i]) s--;
if(s>Ss) { Ss=s; lens=i+1; };
};
lenf+=lens-overlap;
lenb-=lens;
};
for(i=0;i<lenf;i++)
db[dblen+i]=new[lastscan+i]-old[lastpos+i];
for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
eb[eblen+i]=new[lastscan+lenf+i];
dblen+=lenf;
eblen+=(scan-lenb)-(lastscan+lenf);
offtout(lenf,buf);
if(write(p[1],buf,8)!=8) err(1,NULL);
offtout((scan-lenb)-(lastscan+lenf),buf);
if(write(p[1],buf,8)!=8) err(1,NULL);
offtout((pos-lenb)-(lastpos+lenf),buf);
if(write(p[1],buf,8)!=8) err(1,NULL);
lastscan=scan-lenb;
lastpos=pos-lenb;
lastoffset=pos-scan;
};
};
if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
if((len=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
offtout(len-32,buf);
if((lseek(fd,8,SEEK_SET)!=8) || (write(fd,buf,8)!=8))
err(1,"%s",argv[3]);
offtout(newsize,buf);
if((lseek(fd,24,SEEK_SET)!=24) || (write(fd,buf,8)!=8))
err(1,"%s",argv[3]);
if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
if(pid==0) {
if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
(dup2(p[0],0)==-1) || (close(fd)==-1) ||
(close(p[0])==-1) || (close(p[1])==-1))
err(1,NULL);
execl(BZIP2,BZIP2,"-zc",NULL);
err(1,"%s",BZIP2);
};
if(close(p[0])==-1) err(1,NULL);
if(write(p[1],db,dblen)!=dblen) err(1,NULL);
if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
if((newsize=lseek(fd,0,SEEK_END))==-1) err(1,"%s",argv[3]);
offtout(newsize-len,buf);
if((lseek(fd,16,SEEK_SET)!=16) || (write(fd,buf,8)!=8))
err(1,"%s",argv[3]);
if(lseek(fd,0,SEEK_END)==-1) err(1,"%s",argv[3]);
if((pipe(p)==-1) || ((pid=fork())==-1)) err(1,NULL);
if(pid==0) {
if((close(0)==-1) || (close(1)==-1) || (dup2(fd,1)==-1) ||
(dup2(p[0],0)==-1) || (close(fd)==-1) ||
(close(p[0])==-1) || (close(p[1])==-1))
err(1,NULL);
execl(BZIP2,BZIP2,"-zc",NULL);
err(1,"%s",BZIP2);
};
if(close(p[0])==-1) err(1,NULL);
if(write(p[1],eb,eblen)!=eblen) err(1,NULL);
if((close(p[1])==-1) || (waitpid(pid,NULL,0)!=pid)) err(1,NULL);
if(close(fd)==-1) err(1,"%s",argv[3]);
free(db);
free(eb);
free(I);
free(old);
free(new);
return 0;
}
-32
View File
@@ -1,32 +0,0 @@
.Dd May 18, 2003
.Dt BSPATCH 1
.Os FreeBSD
.Sh NAME
.Nm bspatch
.Nd apply a patch built with bsdiff(1)
.Sh SYNOPSIS
.Nm
.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac
.Sh DESCRIPTION
.Nm
generates
.Ao Ar newfile Ac
from
.Ao Ar oldfile Ac
and
.Ao Ar patchfile Ac
where
.Ao Ar patchfile Ac
is a binary patch built by bsdiff(1).
.Pp
.Nm
uses memory equal to the size of
.Ao Ar oldfile Ac
plus the size of
.Ao Ar newfile Ac ,
but can tolerate a very small working set without a dramatic loss
of performance.
.Sh SEE ALSO
.Xr bsdiff 1
.Sh AUTHORS
.An Colin Percival Aq cperciva@daemonology.net
-216
View File
@@ -1,216 +0,0 @@
/*
.c -- Binary patcher
Copyright 2003,2004 Colin Percival
For the terms under which this work may be distributed, please see
the adjoining file "LICENSE".
*/
#ifndef BZIP2
#define BZIP2 "/usr/bin/bzip2"
#endif
#include <stdlib.h>
#include <stdio.h>
#include <err.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/wait.h>
ssize_t loopread(int d,void *buf,size_t nbytes)
{
ssize_t ptr,lenread;
for(ptr=0;ptr<nbytes;ptr+=lenread) {
lenread=read(d,buf+ptr,nbytes-ptr);
if(lenread==0) return ptr;
if(lenread==-1) return -1;
};
return ptr;
}
int bz2read(int fd,off_t offset,off_t len,char * fname,pid_t * pids)
{
int p0[2],p1[2];
u_char * data;
if((pipe(p0)==-1) || (pipe(p1)==-1)) err(1,NULL);
if((pids[0]=fork())==-1) err(1,NULL);
if(pids[0]==0) {
if(close(0) || close(1) || close(p0[0]) ||
close(p1[0]) || close(p1[1])) err(1,NULL);
if((data=malloc(len+1))==NULL) err(1,NULL);
if((pread(fd,data,len,offset)!=len) || close(fd))
err(1,"%s",fname);
if((write(p0[1],data,len)!=len) || close(p0[1]))
err(1,NULL);
free(data);
_exit(0);
};
if((pids[1]=fork())==-1) err(1,NULL);
if(pids[1]==0) {
if(close(0) || close(1) || close(p0[1]) ||
close(p1[0])) err(1,NULL);
if((dup2(p0[0],0)==-1) || close(p0[0])) err(1,NULL);
if((dup2(p1[1],1)==-1) || close(p1[1])) err(1,NULL);
if(close(fd)==-1) err(1,"%s",fname);
execl(BZIP2,BZIP2,"-dc",NULL);
err(1,"%s",BZIP2);
};
if(close(p0[0]) || close(p0[1]) || close(p1[1])) err(1,NULL);
return p1[0];
}
off_t offtin(u_char *buf)
{
off_t y;
y=buf[7]&0x7F;
y=y*256;y+=buf[6];
y=y*256;y+=buf[5];
y=y*256;y+=buf[4];
y=y*256;y+=buf[3];
y=y*256;y+=buf[2];
y=y*256;y+=buf[1];
y=y*256;y+=buf[0];
if(buf[7]&0x80) y=-y;
return y;
}
int main(int argc,char * argv[])
{
int fd,ctrlpipe,diffpipe,extrapipe;
pid_t pids[6];
ssize_t patchsize,oldsize,newsize;
ssize_t bzctrllen,bzdatalen;
u_char header[32],buf[8];
int version=0;
u_char *old, *new;
off_t oldpos,newpos;
off_t ctrl[3];
off_t lenread;
off_t i;
if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]);
if(((fd=open(argv[3],O_RDONLY,0))<0) ||
((patchsize=lseek(fd,0,SEEK_END))==-1) ||
(lseek(fd,0,SEEK_SET)!=0)) err(1,"%s",argv[3]);
if(patchsize<32) errx(1,"Corrupt patch\n");
/*
Ok, this is going to be messy. There are two different patch
formats which we need to support.
The old format (pre-4.0) is:
0 8 "QSUFDIFF" or "BSDIFF30"
8 8 X
16 8 Y
24 8 sizeof(newfile)
32 X bzip2(control block)
32+X Y bzip2(data block)
with control block a set of pairs (x,y) meaning "seek forward
in oldfile by y bytes, and add the next x bytes to x bytes from
the data block".
The new format (4.0) is:
0 8 "BSDIFF40"
8 8 X
16 8 Y
24 8 sizeof(newfile)
32 X bzip2(control block)
32+X Y bzip2(diff block)
32+X+Y ??? bzip2(extra block)
with control block a set of triples (x,y,z) meaning "add x bytes
from oldfile to x bytes from the diff block; copy y bytes from the
extra block; seek forwards in oldfile by z bytes".
*/
if(read(fd,header,32)!=32) err(1,"%s",argv[3]);
if(memcmp(header,"QSUFDIFF",8)==0) version=1;
if(memcmp(header,"BSDIFF30",8)==0) version=1;
if(memcmp(header,"BSDIFF40",8)==0) version=2;
if(!version) errx(1,"Corrupt patch\n");
bzctrllen=offtin(header+8);
bzdatalen=offtin(header+16);
newsize=offtin(header+24);
if((bzctrllen<0) || (bzdatalen<0) || (newsize<0) ||
((version==1) && (32+bzctrllen+bzdatalen!=patchsize)))
errx(1,"Corrupt patch\n");
ctrlpipe=bz2read(fd,32,bzctrllen,argv[3],pids);
diffpipe=bz2read(fd,32+bzctrllen,bzdatalen,argv[3],pids+2);
if(version==2) {
extrapipe=bz2read(fd,32+bzctrllen+bzdatalen,
patchsize-(32+bzctrllen+bzdatalen),argv[3],pids+4);
};
if(close(fd)==-1) err(1,"%s",argv[3]);
if(((fd=open(argv[1],O_RDONLY,0))<0) ||
((oldsize=lseek(fd,0,SEEK_END))==-1) ||
((old=malloc(oldsize+1))==NULL) ||
(lseek(fd,0,SEEK_SET)!=0) ||
(read(fd,old,oldsize)!=oldsize) ||
(close(fd)==-1)) err(1,"%s",argv[1]);
if((new=malloc(newsize+1))==NULL) err(1,NULL);
oldpos=0;newpos=0;
while(newpos<newsize) {
for(i=0;i<=version;i++) {
if((lenread=loopread(ctrlpipe,buf,8))<0) err(1,NULL);
if(lenread<8) errx(1,"Corrupt patch\n");
ctrl[i]=offtin(buf);
};
if(version==1) oldpos+=ctrl[1];
if(newpos+ctrl[0]>newsize) errx(1,"Corrupt patch\n");
if((lenread=loopread(diffpipe,new+newpos,ctrl[0]))<0)
err(1,NULL);
if(lenread!=ctrl[0]) errx(1,"Corrupt patch\n");
for(i=0;i<ctrl[0];i++)
if((oldpos+i>=0) && (oldpos+i<oldsize))
new[newpos+i]+=old[oldpos+i];
newpos+=ctrl[0];
oldpos+=ctrl[0];
if(version==2) {
if(newpos+ctrl[1]>newsize) errx(1,"Corrupt patch\n");
if((lenread=loopread(extrapipe,new+newpos,ctrl[1]))<0)
err(1,NULL);
if(lenread!=ctrl[1]) errx(1,"Corrupt patch\n");
newpos+=ctrl[1];
oldpos+=ctrl[2];
};
};
if(loopread(ctrlpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
if(loopread(diffpipe,buf,1)!=0) errx(1,"Corrupt patch\n");
if(version==2)
if(loopread(extrapipe,buf,1)!=0) errx(1,"Corrupt patch\n");
if(close(ctrlpipe) || close(diffpipe) ||
((version==2) && close(extrapipe)))
err(1,NULL);
for(i=0;i<(version+1)*2;i++) waitpid(pids[i],NULL,0);
if(((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) ||
(write(fd,new,newsize)!=newsize) || (close(fd)==-1))
err(1,"%s",argv[2]);
free(new);
free(old);
return 0;
}
+39 -64
View File
@@ -5,11 +5,9 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <unistd.h>
#include <errno.h>
#include <utime.h>
#include "normalise.hh"
#include "references.hh"
@@ -300,60 +298,6 @@ Path queryNormalForm(const Path & nePath)
}
/* "Fix", or canonicalise, the meta-data of the files in a store path
after it has been built. In particular:
- the last modification date on each file is set to 0 (i.e.,
00:00:00 1/1/1970 UTC)
- the permissions are set of 444 or 555 (i.e., read-only with or
without execute permission; setuid bits etc. are cleared)
- the owner and group are set to the Nix user and group, if we're
in a setuid Nix installation
*/
void canonicalisePathMetaData(const Path & path)
{
checkInterrupt();
struct stat st;
if (lstat(path.c_str(), &st))
throw SysError(format("getting attributes of path `%1%'") % path);
if (!S_ISLNK(st.st_mode)) {
/* Mask out all type related bits. */
mode_t mode = st.st_mode & ~S_IFMT;
if (mode != 0444 && mode != 0555) {
mode = (st.st_mode & S_IFMT)
| 0444
| (st.st_mode & S_IXUSR ? 0111 : 0);
if (chmod(path.c_str(), mode) == -1)
throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
}
if (st.st_uid != getuid() || st.st_gid != getgid()) {
if (chown(path.c_str(), getuid(), getgid()) == -1)
throw SysError(format("changing owner/group of `%1%' to %2%/%3%")
% path % getuid() % getgid());
}
if (st.st_mtime != 0) {
struct utimbuf utimbuf;
utimbuf.actime = st.st_atime;
utimbuf.modtime = 0;
if (utime(path.c_str(), &utimbuf) == -1)
throw SysError(format("changing modification time of `%1%'") % path);
}
}
if (S_ISDIR(st.st_mode)) {
Strings names = readDirectory(path);
for (Strings::iterator i = names.begin(); i != names.end(); ++i)
canonicalisePathMetaData(path + "/" + *i);
}
}
//////////////////////////////////////////////////////////////////////
@@ -1065,7 +1009,7 @@ void NormalisationGoal::createClosure()
}
nf.closure.roots.insert(path);
canonicalisePathMetaData(path);
makePathReadOnly(path);
/* For this output path, find the references to other paths contained
in it. */
@@ -1417,6 +1361,9 @@ private:
/* The current substitute. */
Substitute sub;
/* The normal form of the substitute store expression. */
Path nfSub;
/* Pipe for the substitute's standard output/error. */
Pipe logPipe;
@@ -1438,6 +1385,8 @@ public:
/* The states. */
void init();
void tryNext();
void exprNormalised();
void exprRealised();
void tryToRun();
void finished();
@@ -1502,7 +1451,26 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
/* Wait until we can run the substitute program. */
/* Realise the substitute store expression. */
nrFailed = 0;
addWaitee(worker.makeRealisationGoal(sub.storeExpr));
state = &SubstitutionGoal::exprRealised;
}
void SubstitutionGoal::exprRealised()
{
trace("substitute store expression realised");
if (nrFailed != 0) {
tryNext();
return;
}
/* !!! the storeExpr doesn't have to be a derivation, right? */
nfSub = queryNormalForm(sub.storeExpr);
state = &SubstitutionGoal::tryToRun;
worker.waitForBuildSlot(shared_from_this());
}
@@ -1534,8 +1502,15 @@ void SubstitutionGoal::tryToRun()
printMsg(lvlInfo,
format("substituting path `%1%' using substituter `%2%'")
% storePath % sub.program);
% storePath % sub.storeExpr);
/* What's the substitute program? */
StoreExpr expr = storeExprFromPath(nfSub);
assert(expr.type == StoreExpr::neClosure);
assert(!expr.closure.roots.empty());
Path program =
canonPath(*expr.closure.roots.begin() + "/" + sub.program);
logPipe.create();
/* Remove the (stale) output path if it exists. */
@@ -1561,12 +1536,12 @@ void SubstitutionGoal::tryToRun()
/* Fill in the arguments. */
Strings args(sub.args);
args.push_front(storePath);
args.push_front(baseNameOf(sub.program));
args.push_front(baseNameOf(program));
const char * * argArr = strings2CharPtrs(args);
execv(sub.program.c_str(), (char * *) argArr);
execv(program.c_str(), (char * *) argArr);
throw SysError(format("executing `%1%'") % sub.program);
throw SysError(format("executing `%1%'") % program);
} catch (exception & e) {
cerr << format("substitute error: %1%\n") % e.what();
@@ -1618,7 +1593,7 @@ void SubstitutionGoal::finished()
printMsg(lvlInfo,
format("substitution of path `%1%' using substituter `%2%' failed: %3%")
% storePath % sub.program % e.msg());
% storePath % sub.storeExpr % e.msg());
/* Try the next substitute. */
state = &SubstitutionGoal::tryNext;
@@ -1626,7 +1601,7 @@ void SubstitutionGoal::finished()
return;
}
canonicalisePathMetaData(storePath);
makePathReadOnly(storePath);
Transaction txn;
createStoreTransaction(txn);
+88 -54
View File
@@ -41,13 +41,12 @@ static TableId dbSuccessors = 0;
*/
static TableId dbSuccessorsRev = 0;
/* dbSubstitutes :: Path -> [[Path]]
/* dbSubstitutes :: Path -> [(Path, Path, [string])]
Each pair $(p, subs)$ tells Nix that it can use any of the
substitutes in $subs$ to build path $p$. Each substitute defines a
command-line invocation of a program (i.e., the first list element
is the full path to the program, the remaining elements are
arguments).
substitutes in $subs$ to build path $p$. Each substitute is a
tuple $(storeExpr, program, args)$ (see the type `Substitute' in
`store.hh').
The main purpose of this is for distributed caching of derivates.
One system can compute a derivate and put it on a website (as a Nix
@@ -57,10 +56,18 @@ static TableId dbSuccessorsRev = 0;
*/
static TableId dbSubstitutes = 0;
/* dbSubstitutesRev :: Path -> [Path]
The reverse mapping of dbSubstitutes; it maps store expressions
back to the paths for which they are substitutes.
*/
static TableId dbSubstitutesRev = 0;
bool Substitute::operator == (const Substitute & sub)
{
return program == sub.program
return storeExpr == sub.storeExpr
&& program == sub.program
&& args == sub.args;
}
@@ -79,6 +86,7 @@ void openDB()
dbSuccessors = nixDB.openTable("successors");
dbSuccessorsRev = nixDB.openTable("successors-rev");
dbSubstitutes = nixDB.openTable("substitutes");
dbSubstitutesRev = nixDB.openTable("substitutes-rev");
}
@@ -292,13 +300,10 @@ static Substitutes readSubstitutes(const Transaction & txn,
break;
}
Strings ss2 = unpackStrings(*i);
if (ss2.size() == 3) {
/* Another old-style substitute. */
continue;
}
if (ss2.size() != 2) throw Error("malformed substitute");
if (ss2.size() != 3) throw Error("malformed substitute");
Strings::iterator j = ss2.begin();
Substitute sub;
sub.storeExpr = *j++;
sub.program = *j++;
sub.args = unpackStrings(*j++);
subs.push_back(sub);
@@ -317,6 +322,7 @@ static void writeSubstitutes(const Transaction & txn,
i != subs.end(); ++i)
{
Strings ss2;
ss2.push_back(i->storeExpr);
ss2.push_back(i->program);
ss2.push_back(packStrings(i->args));
ss.push_back(packStrings(ss2));
@@ -326,9 +332,14 @@ static void writeSubstitutes(const Transaction & txn,
}
typedef map<Path, Paths> SubstitutesRev;
void registerSubstitutes(const Transaction & txn,
const SubstitutePairs & subPairs)
{
SubstitutesRev revMap;
for (SubstitutePairs::const_iterator i = subPairs.begin();
i != subPairs.end(); ++i)
{
@@ -336,6 +347,7 @@ void registerSubstitutes(const Transaction & txn,
const Substitute & sub(i->second);
assertStorePath(srcPath);
assertStorePath(sub.storeExpr);
Substitutes subs = readSubstitutes(txn, srcPath);
@@ -345,7 +357,19 @@ void registerSubstitutes(const Transaction & txn,
subs.push_front(sub);
writeSubstitutes(txn, srcPath, subs);
Paths & revs = revMap[sub.storeExpr];
if (revs.empty())
nixDB.queryStrings(txn, dbSubstitutesRev, sub.storeExpr, revs);
if (find(revs.begin(), revs.end(), srcPath) == revs.end())
revs.push_back(srcPath);
}
/* Re-write the reverse mapping in one go to prevent Theta(n^2)
performance. (This would occur because the data fields of the
`substitutes-rev' table are lists). */
for (SubstitutesRev::iterator i = revMap.begin(); i != revMap.end(); ++i)
nixDB.setStrings(txn, dbSubstitutesRev, i->first, i->second);
}
@@ -355,44 +379,6 @@ Substitutes querySubstitutes(const Path & srcPath)
}
static void unregisterPredecessors(const Path & path, Transaction & txn)
{
/* Remove any successor mappings to this path (but not *from*
it). */
Paths revs;
nixDB.queryStrings(txn, dbSuccessorsRev, path, revs);
for (Paths::iterator i = revs.begin(); i != revs.end(); ++i)
nixDB.delPair(txn, dbSuccessors, *i);
nixDB.delPair(txn, dbSuccessorsRev, path);
}
void clearSubstitutes()
{
Transaction txn(nixDB);
/* Iterate over all paths for which there are substitutes. */
Paths subKeys;
nixDB.enumTable(txn, dbSubstitutes, subKeys);
for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
/* If this path has not become valid in the mean-while, delete
any successor mappings *to* it. This is to preserve the
invariant the all successors are `usable' as opposed to
`valid' (i.e., the successor must be valid *or* have at
least one substitute). */
if (!isValidPath(*i)) {
unregisterPredecessors(*i, txn);
}
/* Delete all substitutes for path *i. */
nixDB.delPair(txn, dbSubstitutes, *i);
}
txn.commit();
}
void registerValidPath(const Transaction & txn, const Path & _path)
{
Path path(canonPath(_path));
@@ -407,7 +393,36 @@ static void invalidatePath(const Path & path, Transaction & txn)
debug(format("unregistering path `%1%'") % path);
nixDB.delPair(txn, dbValidPaths, path);
unregisterPredecessors(path, txn);
/* Remove any successor mappings to this path (but not *from*
it). */
Paths revs;
nixDB.queryStrings(txn, dbSuccessorsRev, path, revs);
for (Paths::iterator i = revs.begin(); i != revs.end(); ++i)
nixDB.delPair(txn, dbSuccessors, *i);
nixDB.delPair(txn, dbSuccessorsRev, path);
/* Remove any substitute mappings to this path. */
revs.clear();
nixDB.queryStrings(txn, dbSubstitutesRev, path, revs);
for (Paths::iterator i = revs.begin(); i != revs.end(); ++i) {
Substitutes subs = readSubstitutes(txn, *i), subs2;
bool found = false;
for (Substitutes::iterator j = subs.begin(); j != subs.end(); ++j)
if (j->storeExpr != path)
subs2.push_back(*j);
else
found = true;
// !!! if (!found) throw Error("integrity error in substitutes mapping");
writeSubstitutes(txn, *i, subs);
/* If path *i now has no substitutes left, and is not valid,
then it too should be invalidated. This is because it may
be a substitute or successor. */
if (subs.size() == 0 && !isValidPathTxn(*i, txn))
invalidatePath(*i, txn);
}
nixDB.delPair(txn, dbSubstitutesRev, path);
}
@@ -533,11 +548,30 @@ void verifyStore()
Paths subKeys;
nixDB.enumTable(txn, dbSubstitutes, subKeys);
for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
Substitutes subs = readSubstitutes(txn, *i);
if (subs.size() > 0)
Substitutes subs = readSubstitutes(txn, *i), subs2;
for (Substitutes::iterator j = subs.begin(); j != subs.end(); ++j)
if (validPaths.find(j->storeExpr) == validPaths.end())
printMsg(lvlError,
format("found substitute mapping to non-existent path `%1%'")
% j->storeExpr);
else
subs2.push_back(*j);
if (subs.size() != subs2.size())
writeSubstitutes(txn, *i, subs2);
if (subs2.size() > 0)
usablePaths.insert(*i);
else
nixDB.delPair(txn, dbSubstitutes, *i);
}
/* Check that the keys of the reverse substitute mappings are
valid paths. */
Paths rsubKeys;
nixDB.enumTable(txn, dbSubstitutesRev, rsubKeys);
for (Paths::iterator i = rsubKeys.begin(); i != rsubKeys.end(); ++i) {
if (validPaths.find(*i) == validPaths.end()) {
printMsg(lvlError,
format("found reverse substitute mapping for non-existent path `%1%'") % *i);
nixDB.delPair(txn, dbSubstitutesRev, *i);
}
}
/* Check that the values of the successor mappings are usable
+4 -3
View File
@@ -14,6 +14,10 @@ using namespace std;
network). */
struct Substitute
{
/* Store expression to be normalised and realised in order to
obtain `program'. */
Path storeExpr;
/* Program to be executed to create the store path. Must be in
the output path of `storeExpr'. */
Path program;
@@ -69,9 +73,6 @@ void registerSubstitutes(const Transaction & txn,
/* Return the substitutes expression for the given path. */
Substitutes querySubstitutes(const Path & srcPath);
/* Deregister all substitutes. */
void clearSubstitutes();
/* Register the validity of a path. */
void registerValidPath(const Transaction & txn, const Path & path);
-1
View File
@@ -11,7 +11,6 @@ Operations:
--successor: register a successor expression (dangerous!)
--substitute: register a substitute expression (dangerous!)
--clear-substitute: clear all substitutes
--validpath: register path validity (dangerous!)
--isvalid: check path validity
+1 -13
View File
@@ -166,6 +166,7 @@ static void opSubstitute(Strings opFlags, Strings opArgs)
Substitute sub;
getline(cin, srcPath);
if (cin.eof()) break;
getline(cin, sub.storeExpr);
getline(cin, sub.program);
string s;
getline(cin, s);
@@ -185,17 +186,6 @@ static void opSubstitute(Strings opFlags, Strings opArgs)
}
static void opClearSubstitutes(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
if (!opArgs.empty())
throw UsageError("no arguments expected");
clearSubstitutes();
}
static void opValidPath(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
@@ -364,8 +354,6 @@ void run(Strings args)
op = opSuccessor;
else if (arg == "--substitute")
op = opSubstitute;
else if (arg == "--clear-substitutes")
op = opClearSubstitutes;
else if (arg == "--validpath")
op = opValidPath;
else if (arg == "--isvalid")
+1 -1
View File
@@ -4,7 +4,7 @@ echo "store expr is $storeExpr"
# Register a non-existant successor (and a nox-existant substitute).
suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store
(echo $suc && echo $TOP/no-such-program && echo 0) | $TOP/src/nix-store/nix-store --substitute
(echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr")
+2 -2
View File
@@ -10,7 +10,7 @@ while ! ln -s x $shared.lock; do
done
test -f $shared.cur || echo 0 > $shared.cur
test -f $shared.max || echo 0 > $shared.max
new=$(expr $(cat $shared.cur) + 1)
new=$(($(cat $shared.cur) + 1))
if test $new -gt $(cat $shared.max); then
echo $new > $shared.max
fi
@@ -28,5 +28,5 @@ while ! ln -s x $shared.lock; do
sleep 1
done
test -f $shared.cur || echo 0 > $shared.cur
echo $(expr $(cat $shared.cur) - 1) > $shared.cur
echo $(($(cat $shared.cur) - 1)) > $shared.cur
rm $shared.lock
+6 -7
View File
@@ -6,22 +6,21 @@ echo "store expr is $storeExpr"
outPath=$($TOP/src/nix-store/nix-store -qvvvvv "$storeExpr")
echo "output path is $outPath"
# Build the substitute program.
subProgram=$($TOP/src/nix-store/nix-store -qnf \
$($TOP/src/nix-instantiate/nix-instantiate substituter.nix))/substituter
echo "substitute program is $subProgram"
# Instantiate the substitute program.
subExpr=$($TOP/src/nix-instantiate/nix-instantiate substituter.nix)
echo "store expr is $subExpr"
regSub() {
(echo $1 && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
(echo $1 && echo $2 && echo "/substituter" && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
}
# Register a fake successor, and a substitute for it.
suc=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-s.store
regSub $suc $subProgram
regSub $suc $subExpr
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
# Register a substitute for the output path.
regSub $outPath $subProgram
regSub $outPath $subExpr
$TOP/src/nix-store/nix-store -rvvvvv "$storeExpr"
+11 -13
View File
@@ -6,34 +6,32 @@ echo "store expr is $storeExpr"
outPath=$($TOP/src/nix-store/nix-store -qvvvvv "$storeExpr")
echo "output path is $outPath"
# Build the substitute program.
subProgram=$($TOP/src/nix-store/nix-store -qnf \
$($TOP/src/nix-instantiate/nix-instantiate substituter.nix))/substituter
echo "substitute program is $subProgram"
# Instantiate the substitute program.
subExpr=$($TOP/src/nix-instantiate/nix-instantiate substituter.nix)
echo "store expr is $subExpr"
# Build the failing substitute program.
subProgram2=$($TOP/src/nix-store/nix-store -qnf \
$($TOP/src/nix-instantiate/nix-instantiate substituter2.nix))/substituter
echo "failing substitute program is $subProgram2"
# Instantiate the failing substitute program.
subExpr2=$($TOP/src/nix-instantiate/nix-instantiate substituter2.nix)
echo "store expr is $subExpr2"
regSub() {
(echo $1 && echo $2 && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
(echo $1 && echo $2 && echo "/substituter" && echo 3 && echo $outPath && echo Hallo && echo Wereld) | $TOP/src/nix-store/nix-store --substitute
}
# Register a fake successor, and a substitute for it.
suc=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab-s.store
regSub $suc $subProgram
regSub $suc $subExpr
$TOP/src/nix-store/nix-store --successor $storeExpr $suc
# Register a failing substitute for it (it takes precedence).
regSub $suc $subProgram2
regSub $suc $subExpr2
# Register a substitute for the output path.
regSub $outPath $subProgram
regSub $outPath $subExpr
# Register another substitute for the output path. This one will
# produce other output.
regSub $outPath $subProgram2
regSub $outPath $subExpr2
$TOP/src/nix-store/nix-store -rvvvvv "$storeExpr"