2000-09-16 00:05:46 +02:00
|
|
|
#!/usr/pkg/bin/perl
|
2002-06-29 05:33:34 +02:00
|
|
|
# $NetBSD: post-build,v 1.18 2002/06/29 03:33:34 dmcmahill Exp $
|
2000-09-16 00:05:46 +02:00
|
|
|
#
|
|
|
|
|
# Collect stuff after a pkg bulk build
|
|
|
|
|
#
|
|
|
|
|
# (c) 2000 Hubert Feyrer, All Rights Reserved.
|
|
|
|
|
#
|
|
|
|
|
|
|
|
|
|
use File::Basename;
|
|
|
|
|
|
|
|
|
|
# Where config vars are stored (/bin/sh syntax)
|
|
|
|
|
|
|
|
|
|
if (-f $ENV{"BULK_BUILD_CONF"}) {
|
|
|
|
|
$BULK_BUILD_CONF=$ENV{"BULK_BUILD_CONF"};
|
|
|
|
|
} else {
|
|
|
|
|
$BULK_BUILD_CONF=dirname("$0")."/build.conf";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Dig given variable out of config file, and set it
|
|
|
|
|
sub getconf
|
|
|
|
|
{
|
|
|
|
|
local($var)=@_;
|
|
|
|
|
local($val);
|
|
|
|
|
|
|
|
|
|
chomp($val=`. ./$BULK_BUILD_CONF ; echo \$$var`);
|
|
|
|
|
eval "\$$var=\"$val\";";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2001-03-19 12:25:39 +01:00
|
|
|
getconf("ADMINSIG"); # "-Your Name"
|
2000-09-16 00:05:46 +02:00
|
|
|
getconf("FTPURL"); # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
|
|
|
|
|
getconf("FTP"); # "/disk1/ftp/${FTPURL}"
|
2000-09-17 23:51:09 +02:00
|
|
|
getconf("FTPHOST"); # ftp://ftp.machi.ne/
|
2000-09-16 00:05:46 +02:00
|
|
|
getconf("REPORT"); # "broken.html"
|
|
|
|
|
getconf("USR_PKGSRC"); # "/usr/pkgsrc"
|
|
|
|
|
getconf("osrev"); # `uname -r`
|
|
|
|
|
getconf("arch"); # `uname -m`
|
|
|
|
|
chomp($date=`date`);
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
$reportf=basename($REPORT);
|
2000-09-16 00:05:46 +02:00
|
|
|
|
2000-12-31 18:38:32 +01:00
|
|
|
# extract the name of the files used for the build log and broken build log.
|
|
|
|
|
# these have defaults set by bsd.bulk-pkg.mk and may be overridden in /etc/mk.conf
|
|
|
|
|
chomp($BROKENFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=BROKENFILE )`);
|
2000-09-16 00:05:46 +02:00
|
|
|
|
2001-05-03 23:46:32 +02:00
|
|
|
# also extract the names of the cache files used during the bulk build. We'll save a copy of
|
|
|
|
|
# those to help debug if the build messed up.
|
|
|
|
|
chomp($DEPENDSTREEFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=DEPENDSTREEFILE )`);
|
|
|
|
|
chomp($DEPENDSFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=DEPENDSFILE )`);
|
|
|
|
|
chomp($SUPPORTSFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=SUPPORTSFILE )`);
|
|
|
|
|
chomp($INDEXFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=INDEXFILE )`);
|
|
|
|
|
chomp($ORDERFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=ORDERFILE )`);
|
2002-06-29 05:33:34 +02:00
|
|
|
chomp($STARTFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=STARTFILE )`);
|
|
|
|
|
chomp($LOCALBASE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=LOCALBASE )`);
|
|
|
|
|
chomp($X11BASE=`( cd $USR_PKGSRC/pkgtools/pkglint ; make show-var VARNAME=X11BASE )`);
|
2001-05-03 23:46:32 +02:00
|
|
|
$dtfile=basename($DEPENDSTREEFILE);
|
|
|
|
|
$depfile=basename($DEPENDSFILE);
|
|
|
|
|
$supfile=basename($SUPPORTSFILE);
|
|
|
|
|
$indfile=basename($INDEXFILE);
|
|
|
|
|
$ordfile=basename($ORDERFILE);
|
2002-06-29 05:33:34 +02:00
|
|
|
$startfile=basename($STARTFILE);
|
2001-05-03 23:46:32 +02:00
|
|
|
|
2000-12-31 18:38:32 +01:00
|
|
|
$verbose=1;
|
2000-09-16 00:05:46 +02:00
|
|
|
|
|
|
|
|
if ($verbose) {
|
|
|
|
|
print "\n";
|
|
|
|
|
print "*** NetBSD $osrev/$arch\n";
|
|
|
|
|
print "*** Result of bulk pkgsrc build as of $date:\n";
|
|
|
|
|
print "\n";
|
2001-10-16 01:18:51 +02:00
|
|
|
printf ("%-23s State \tBreaks\t(last modified, maintainer)\n","Pkg");
|
2000-09-16 00:05:46 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
system("mkdir -p ${FTP}");
|
|
|
|
|
|
|
|
|
|
# Copy over the output from the build process
|
|
|
|
|
chdir("$USR_PKGSRC");
|
2000-12-26 17:45:57 +01:00
|
|
|
system("tar plcf - $BROKENFILE */*/$BROKENFILE | ( cd ${FTP} ; tar plxf - )");
|
2000-09-16 00:05:46 +02:00
|
|
|
|
2001-05-03 23:46:32 +02:00
|
|
|
# Copy over the cache files used during the build
|
|
|
|
|
if (-f $DEPENDSTREEFILE ) {system("cp $DEPENDSTREEFILE ${FTP}");}
|
|
|
|
|
if (-f $DEPENDSFILE ) {system("cp $DEPENDSFILE ${FTP}");}
|
|
|
|
|
if (-f $SUPPORTSFILE ) {system("cp $SUPPORTSFILE ${FTP}");}
|
|
|
|
|
if (-f $INDEXFILE ) {system("cp $INDEXFILE ${FTP}");}
|
|
|
|
|
if (-f $ORDERFILE ) {system("cp $ORDERFILE ${FTP}");}
|
|
|
|
|
|
2000-09-16 00:05:46 +02:00
|
|
|
|
2000-12-30 14:05:37 +01:00
|
|
|
open(HTML,">$REPORT") or die "Can't write $REPORT: $!\n";
|
2000-09-16 00:05:46 +02:00
|
|
|
print HTML <<EOHTML
|
|
|
|
|
<HTML>
|
|
|
|
|
<HEAD>
|
|
|
|
|
<TITLE>NetBSD-$osrev/$arch bulk package build</TITLE>
|
|
|
|
|
</HEAD>
|
|
|
|
|
|
|
|
|
|
<BODY BGCOLOR=#cccccc>
|
|
|
|
|
|
|
|
|
|
<H1> NetBSD $osrev/$arch </H1>
|
|
|
|
|
<H2>Output of the pkgsrc bulk build<BR>
|
|
|
|
|
as of $date</H2>
|
|
|
|
|
|
|
|
|
|
Packages not listed here resulted in a <A
|
|
|
|
|
HREF="../../packages/">binary package</A>. To see the output of
|
|
|
|
|
the (failed) build, select the package below.
|
|
|
|
|
<P>
|
|
|
|
|
|
|
|
|
|
Files not listed in PLISTs can be found in <A HREF="leftovers-${arch}.html">this list</A>.
|
|
|
|
|
<P>
|
|
|
|
|
|
|
|
|
|
<TABLE>
|
2001-02-08 20:17:38 +01:00
|
|
|
<TR><TH> Package <TH> State <TH> Pkgs Broken by <TH> Last touched by <TH> File touched last <TH> Maintainer
|
2000-09-16 00:05:46 +02:00
|
|
|
EOHTML
|
|
|
|
|
;
|
|
|
|
|
|
|
|
|
|
# some nifty postprocessing of $FTP/.broken* -> HTML here
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
$nunpackaged = $nbroken = $nbrokendep = 0;
|
2000-09-16 00:05:46 +02:00
|
|
|
chdir("$FTP");
|
2000-12-26 17:45:57 +01:00
|
|
|
open(BF,"sort +1 $BROKENFILE |") or die "can't open .broken-file '$BROKENFILE'";
|
2000-09-16 00:05:46 +02:00
|
|
|
while (<BF>) {
|
2001-02-08 20:17:38 +01:00
|
|
|
($nerrors, $bf, $nbrokenby) = split;
|
2000-12-26 20:01:35 +01:00
|
|
|
$pkg = $bf;
|
2000-12-26 17:45:57 +01:00
|
|
|
$pkg =~ s,/$BROKENFILE,,;
|
2000-09-16 00:05:46 +02:00
|
|
|
|
|
|
|
|
# next
|
|
|
|
|
# if $pkg!~/^a/; # DEBUG! HF
|
|
|
|
|
|
|
|
|
|
if ($nerrors > 0) {
|
|
|
|
|
$color = "red";
|
|
|
|
|
$state = "build broken";
|
|
|
|
|
$nbroken++;
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
} elsif ($nerrors == -1) {
|
|
|
|
|
$color = "orange";
|
|
|
|
|
$state = "broken depends";
|
|
|
|
|
$nbrokendep++;
|
|
|
|
|
} else {
|
2000-09-16 00:05:46 +02:00
|
|
|
$color = "yellow";
|
|
|
|
|
$state = "not packaged";
|
|
|
|
|
$nunpackaged++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# The idea of this is stolen from bsd.pkg.mk's
|
|
|
|
|
# show-pkgtools-version target - thanks Al! :-)
|
|
|
|
|
$whofile = `find ${USR_PKGSRC}/${pkg} -type f -print | xargs ident 2>/dev/null | awk '\$1 ~ /\\\$NetBSD/ { gsub("/", "", \$4); print \$4 " " \$6 " " \$2 }' | sort -n | tail -n 1 | awk '{ print \$2 " " \$3 }' `;
|
|
|
|
|
$whofile=~s/,v$//;
|
|
|
|
|
chomp($whofile);
|
|
|
|
|
($who,$file) = split(/\s+/, $whofile);
|
|
|
|
|
|
|
|
|
|
# $maintainer=`( cd $USR_PKGSRC/$pkg ; ( echo '.include "Makefile"' ; echo 'bla: ; \@echo ${MAINTAINER}' ) | make -f - bla )`;
|
|
|
|
|
$maintainer=`grep ^MAINTAINER $USR_PKGSRC/$pkg/Makefile | awk '{ print \$2\$3\$4\$5\$6}' | sed -e 's/\</</g' -e 's/\>/>/g'`;
|
|
|
|
|
chomp($maintainer);
|
|
|
|
|
|
|
|
|
|
print HTML "<TR><TD><A HREF=\"$bf\">$pkg</A>";
|
|
|
|
|
print HTML " <TD><FONT COLOR=$color>$state</FONT>";
|
2001-02-08 20:17:38 +01:00
|
|
|
if ($nbrokenby > 0){
|
2001-10-16 22:50:54 +02:00
|
|
|
print HTML " <TD ALIGN=\"right\"><FONT COLOR=$color>$nbrokenby</FONT>";
|
2001-02-08 20:17:38 +01:00
|
|
|
}
|
|
|
|
|
else {
|
|
|
|
|
print HTML " <TD> ";
|
|
|
|
|
}
|
2000-09-16 00:05:46 +02:00
|
|
|
print HTML " <TD> $who <TD> $file";
|
|
|
|
|
print HTML " <TD> $maintainer\n";
|
|
|
|
|
|
|
|
|
|
($category, $dbfeed_pkg) = split('/', $pkg);
|
|
|
|
|
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
if ($nerrors != 0) {
|
2001-02-08 20:17:38 +01:00
|
|
|
if ($nbrokenby > 0){
|
2001-10-16 01:18:51 +02:00
|
|
|
printf("%-23s $state\t%-5d\t($who, $maintainer)\n", "$pkg:",$nbrokenby)
|
2001-02-08 20:17:38 +01:00
|
|
|
if $verbose;
|
|
|
|
|
}
|
|
|
|
|
else {
|
2001-10-16 01:18:51 +02:00
|
|
|
printf("%-23s $state\t%-5s\t($who, $maintainer)\n", "$pkg:","")
|
2001-02-08 20:17:38 +01:00
|
|
|
if $verbose;
|
|
|
|
|
}
|
|
|
|
|
|
2000-09-16 00:05:46 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
close(BF);
|
|
|
|
|
|
|
|
|
|
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
$nbrokentot=$nbroken+$nbrokendep;
|
|
|
|
|
$ntotal=$nunpackaged+$nbroken+$nbrokendep;
|
2000-09-16 00:05:46 +02:00
|
|
|
print HTML <<EOHTML
|
|
|
|
|
</TABLE>
|
|
|
|
|
|
2001-05-03 23:46:32 +02:00
|
|
|
<HR>
|
|
|
|
|
<P>
|
|
|
|
|
The following cache files were used during the build:
|
|
|
|
|
<UL>
|
|
|
|
|
<LI>The <a href=$dtfile>depends tree file</a>.
|
|
|
|
|
<LI>The <a href=$depfile>depends file</a>.
|
|
|
|
|
<LI>The <a href=$supfile>supports file</a>.
|
|
|
|
|
<LI>The <a href=$indfile>index file</a>.
|
|
|
|
|
<LI>The <a href=$ordfile>build order file</a>.
|
|
|
|
|
</UL>
|
|
|
|
|
<P>
|
2000-09-16 00:05:46 +02:00
|
|
|
<HR>
|
|
|
|
|
|
|
|
|
|
<TABLE>
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
<TR><TD> Packages not packaged: <TD ALIGHT=RIGHT> <FONT COLOR=yellow>$nunpackaged</FONT>
|
|
|
|
|
<TR><TD> Packages really broken: <TD ALIGHT=RIGHT> <FONT COLOR=red>$nbroken</FONT>
|
|
|
|
|
<TR><TD> Pkgs broken due to them: <TD ALIGHT=RIGHT> <FONT COLOR=blue>$nbrokendep</FONT>
|
|
|
|
|
<TR><TD> Total broken: <TD ALIGHT=RIGHT> $nbrokentot
|
|
|
|
|
<TR><TD> Total: <TD ALIGHT=RIGHT> $ntotal
|
2000-09-16 00:05:46 +02:00
|
|
|
</TABLE>
|
|
|
|
|
|
|
|
|
|
<HR>
|
|
|
|
|
<P>
|
|
|
|
|
<UL>
|
|
|
|
|
<!-- <LI>View the <A HREF="$PROG">progress</A> of the build. -->
|
|
|
|
|
<!-- <LI>See the list of <A HREF="../index.html">all log files</A>. -->
|
|
|
|
|
<LI>Visit the <A HREF="http://www.netbsd.org">NetBSD Homepage</A>.
|
|
|
|
|
<LI>Learn about the
|
|
|
|
|
<A HREF="http://www.netbsd.org/Documentation/software/packages.html">
|
|
|
|
|
NetBSD Package System</A>.
|
|
|
|
|
</UL>
|
|
|
|
|
</P>
|
|
|
|
|
<HR>
|
|
|
|
|
Hubert Feyrer, 1999-2000.
|
|
|
|
|
</BODY>
|
|
|
|
|
</HTML>
|
|
|
|
|
EOHTML
|
|
|
|
|
;
|
|
|
|
|
close(HTML);
|
|
|
|
|
|
|
|
|
|
#
|
|
|
|
|
# Adjust "last" symlink
|
|
|
|
|
#
|
|
|
|
|
{
|
|
|
|
|
local($base, $dir) = $FTP=~m|^(.*)/([^/]*)$|;
|
|
|
|
|
|
|
|
|
|
unlink("$base/last");
|
|
|
|
|
symlink($dir, "$base/last");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#
|
|
|
|
|
# Generate leftovers-${arch}.html: files not deleted
|
|
|
|
|
# Leftover files are copied to leftovers-${arch} dir, and linked from
|
|
|
|
|
# leftovers-${arch}.html
|
|
|
|
|
#
|
|
|
|
|
{
|
|
|
|
|
chdir("${FTP}");
|
|
|
|
|
system("mkdir -p leftovers-${arch}");
|
|
|
|
|
|
|
|
|
|
# Find files since last build:
|
2002-06-29 05:33:34 +02:00
|
|
|
system("find ${LOCALBASE} -newer ${USR_PKGSRC}/${startfile} -type f >>leftovers-${arch}.txt");
|
|
|
|
|
system("find ${X11BASE} -newer ${USR_PKGSRC}/${startfile} -type f >>leftovers-${arch}.txt");
|
2000-09-16 00:05:46 +02:00
|
|
|
|
|
|
|
|
# Strip perl-files:
|
2001-10-16 01:00:10 +02:00
|
|
|
system("pkg_info -qL perl > .p.${arch}");
|
2000-09-16 00:05:46 +02:00
|
|
|
system("fgrep -v -x -f .p.${arch} leftovers-${arch}.txt >leftovers-${arch}.txt.new");
|
|
|
|
|
rename("leftovers-${arch}.txt.new", "leftovers-${arch}.txt");
|
|
|
|
|
unlink(".p.${arch}");
|
|
|
|
|
|
|
|
|
|
# Store leftovers, for easier identification:
|
|
|
|
|
system("tar plcf - `cat leftovers-${arch}.txt` | ( cd leftovers-${arch} ; tar plxf - )");
|
|
|
|
|
|
|
|
|
|
# Add links to leftover list:
|
|
|
|
|
open (IN, "< leftovers-${arch}.txt")
|
|
|
|
|
or die "can't read leftovers-${arch}.txt";
|
|
|
|
|
open (OUT, "> leftovers-${arch}.html")
|
|
|
|
|
or die "can't write leftovers-${arch}.html";
|
|
|
|
|
print OUT <<EOOUT
|
|
|
|
|
<HTML>
|
|
|
|
|
<BODY>
|
|
|
|
|
<PRE>
|
|
|
|
|
EOOUT
|
|
|
|
|
;
|
|
|
|
|
while (<IN>) {
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
print OUT "<A HREF=\"${FTPHOST}/${FTPURL}/leftovers-${arch}$_\">$_</A>\n";
|
2000-09-16 00:05:46 +02:00
|
|
|
}
|
|
|
|
|
print OUT <<EOOUT2
|
|
|
|
|
</PRE>
|
|
|
|
|
</BODY>
|
|
|
|
|
</HTML>
|
|
|
|
|
EOOUT2
|
|
|
|
|
;
|
|
|
|
|
close(IN);
|
|
|
|
|
close(OUT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($verbose) {
|
|
|
|
|
print "\n";
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
print "Packages not packaged: $nunpackaged\n";
|
|
|
|
|
print "Packages really broken: $nbroken\n";
|
|
|
|
|
print "Pkgs broken due to them: $nbrokendep\n";
|
|
|
|
|
print "Total broken: $nbrokentot\n";
|
|
|
|
|
print "Total: $ntotal\n";
|
2000-09-16 00:05:46 +02:00
|
|
|
print "\n";
|
This commit incorporates several speed improvements which have been tested
over several bulk builds on an alpha:
- At the start of a bulk build, 4 files are created to allow fast lookup of
various dependency tree things. These files are
.index == maps package name (foo-2.3) to directory (bar/foo)
.dependstree == contains the complete pkgsrc dependency tree in a tsort(1)
compatible format.
.depends == contains 1 line per package and lists all the build depends
for the package.
.supports == contains 1 line per package and lists all packages which
depend on this package.
- When a package fails to build, the list of all packages which depend upon
it is read from .supports. Each of those packages is immediately marked
as broken. This prevents us from trying to build those packages which can
have significant overhead if lots of other depends are installed before the
system notices the broken one.
In addition, the post-build postprocessing will now indicate that a package
is 'truely broken' (ie, bad PLIST, failed compilation) vs. a package which
is broken because it depends on a failed package. This assists in determining
where to focus our efforts in fixing broken packages.
- In the old approach, all packages are removed after each one is built. The
purpose was to a) conserve disk space, b) verify that all required dependencies
are in fact listed, and c) prevent conflicts. The problem was that often times
several packages in a row have similar depends. For example we might want to
build several perl packages in a row. In the old approach, we would install perl,
build the pkg, deinstall perl and continue with the next one.
In the new approach, when it is decided that a pkg is out of date and should be
rebuilt, the .depends file is used to obtain a list of pkgs we require. Then
and pkgs which are no longer needed are removed. This helps to minimize the total
number of pkg_add/pkg_delete's in a bulk build.
- Since the order of the build is controlled by the depends tree, all depends for a
given package will have been built by the time the pkg in question needs them.
If any of the depends failed to build, then the pkg which needs the failed one will
have been marked as broken. Given this, the complete depends list for a pkg is
read from .depends and the depends installed via pkg_add rather than relying on
recursive make calls to install the depends.
- while here, fix a few minor bugs
x - don't leave .make log files around when the build succeeds
x - make sure we refer to the correct report file in the email
x - use '.order' for the build order file instead of '.l'
x - use 'grep -c' instead of 'grep | wc -l'
2001-02-01 09:47:29 +01:00
|
|
|
print "See $FTPHOST/$FTPURL/$reportf\n";
|
2000-09-16 00:05:46 +02:00
|
|
|
print "for logs of builds broken or not resulting in a binary pkg.\n";
|
|
|
|
|
print "\n";
|
|
|
|
|
print "\n";
|
2001-03-19 12:25:39 +01:00
|
|
|
print "$ADMINSIG\n";
|
2000-09-16 00:05:46 +02:00
|
|
|
print "\n";
|
|
|
|
|
print "[* This message was created automatically! *]\n";
|
|
|
|
|
print "\n";
|
|
|
|
|
}
|
