365 lines
11 KiB
Bash
365 lines
11 KiB
Bash
#!/bin/sh
|
|
# $NetBSD: build,v 1.48 2005/01/14 15:32:34 tv Exp $
|
|
|
|
#
|
|
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
# 3. All advertising materials mentioning features or use of this software
|
|
# must display the following acknowledgement:
|
|
# This product includes software developed by Hubert Feyrer for
|
|
# the NetBSD Project.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
#
|
|
# Do bulk build
|
|
#
|
|
|
|
usage() {
|
|
echo "Usage:"
|
|
echo " $0 [-m | --mirror_only] [-r | --restart | restart]"
|
|
echo " [-s | --specific-pkgs]"
|
|
echo " $0 -h|--help"
|
|
echo " "
|
|
echo "Runs a bulk pkgsrc build."
|
|
echo " "
|
|
echo "The following options are supported:"
|
|
echo " "
|
|
echo " -h|--help Displays this message."
|
|
echo " "
|
|
echo " -m|--mirror_only Downloads all distfiles need for the"
|
|
echo " build but does not run the build."
|
|
echo " IMPORTANT: Note that this will still"
|
|
echo " run all the pre-build stuff which involves"
|
|
echo " removing all of your installed packages."
|
|
echo " "
|
|
echo " The only difference between this option"
|
|
echo " and a regular bulk build is that the packages"
|
|
echo " are not actually built."
|
|
echo " "
|
|
echo " -r|--restart|restart Restart a previously interrupted bulk build."
|
|
echo " last form of this option is for backwards"
|
|
echo " compatibility and may be removed in future"
|
|
echo " versions of this script."
|
|
echo " "
|
|
echo " The --restart option may be combined with the"
|
|
echo " --mirror_only option."
|
|
echo " "
|
|
echo " -s|--specific-pkgs Sets SPECIFIC_PKGS=1 when building packages."
|
|
echo " This option is used for building a subset of"
|
|
echo " pkgsrc."
|
|
echo " "
|
|
}
|
|
|
|
restart=no
|
|
mirror_only=no
|
|
target=bulk-package
|
|
makeargs=""
|
|
|
|
while [ ${#} -ge 1 ] ; do
|
|
case $1 in
|
|
|
|
--help|-h )
|
|
usage
|
|
exit 0
|
|
;;
|
|
|
|
--mirror_only|-m )
|
|
mirror_only=yes
|
|
target=mirror-distfiles
|
|
shift
|
|
;;
|
|
|
|
--restart|-r|restart )
|
|
restart=yes
|
|
shift
|
|
;;
|
|
|
|
--specific-pkgs|-s )
|
|
makeargs="$makeargs SPECIFIC_PKGS=1"
|
|
shift
|
|
;;
|
|
|
|
-* )
|
|
echo "unknown option: $1"
|
|
usage
|
|
exit 1
|
|
;;
|
|
|
|
* )
|
|
break
|
|
;;
|
|
|
|
esac
|
|
done
|
|
|
|
if [ $# -ne 0 ]; then
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
opsys=`uname -s`
|
|
case "$opsys" in
|
|
NetBSD) BMAKE=make ;;
|
|
*) BMAKE=bmake ;;
|
|
esac
|
|
|
|
export BMAKE
|
|
|
|
# Set resource limits as high as possible
|
|
ulimit -S -s `ulimit -H -s`
|
|
ulimit -S -d `ulimit -H -d`
|
|
|
|
|
|
echo Bulk build started: `date`
|
|
echo ""
|
|
|
|
# Pull in ADMIN etc.:
|
|
if [ -f "$BULK_BUILD_CONF" ]; then
|
|
. $BULK_BUILD_CONF
|
|
else
|
|
conf=`dirname $0`/build.conf
|
|
if [ -f "$conf" ]; then
|
|
. $conf
|
|
else
|
|
echo "$0: Cannot find config file $conf, aborting."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# set up variables specifically for the bulk build
|
|
BATCH=1
|
|
DEPENDS_TARGET=bulk-install
|
|
export BATCH DEPENDS_TARGET
|
|
|
|
if [ "$http_proxy" != "" ]; then
|
|
echo "Using HTTP proxy $http_proxy"
|
|
export http_proxy
|
|
fi
|
|
if [ "$ftp_proxy" != "" ]; then
|
|
echo "Using FTP proxy $ftp_proxy"
|
|
export ftp_proxy
|
|
fi
|
|
echo ""
|
|
|
|
unset CDPATH # ensure cd does not print new cwd to stdout, which
|
|
# confuses the printindex script.
|
|
unset DISPLAY # allow sane failure for gimp, xlispstat
|
|
|
|
# Check that the pkg_tools are up to date
|
|
(cd ${USR_PKGSRC}/pkgtools/pkglint && \
|
|
${BMAKE} fetch >/dev/null 2>&1 ) || \
|
|
( echo "Updating pkgtools" && \
|
|
cd ${USR_PKGSRC}/pkgtools/pkg_install && ${BMAKE} clean && \
|
|
${BMAKE} install && ${BMAKE} clean )
|
|
|
|
cd ${USR_PKGSRC}
|
|
|
|
|
|
if [ "x$restart" = "xyes" ]; then
|
|
echo Restarting - skipping pre-build script
|
|
else
|
|
sh mk/bulk/pre-build # make veryveryclean :)
|
|
if [ $? != 0 ]
|
|
then
|
|
echo "Error during bulk-build preparations, aborting."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
fail=no
|
|
if [ -d pkgtools/pkglint ]; then
|
|
cd pkgtools/pkglint
|
|
PERL5_PKGSRCDIR=`${BMAKE} show-var VARNAME=PERL5_PKGSRCDIR` || fail=yes
|
|
BULK_DBFILE=`${BMAKE} show-var VARNAME=BULK_DBFILE` || fail=yes
|
|
DEPENDSTREEFILE=`${BMAKE} show-var VARNAME=DEPENDSTREEFILE` || fail=yes
|
|
DEPENDSFILE=`${BMAKE} show-var VARNAME=DEPENDSFILE` || fail=yes
|
|
SUPPORTSFILE=`${BMAKE} show-var VARNAME=SUPPORTSFILE` || fail=yes
|
|
INDEXFILE=`${BMAKE} show-var VARNAME=INDEXFILE` || fail=yes
|
|
ORDERFILE=`${BMAKE} show-var VARNAME=ORDERFILE` || fail=yes
|
|
BROKENFILE=`${BMAKE} show-var VARNAME=BROKENFILE` || fail=yes
|
|
BROKENWRKLOG=`${BMAKE} show-var VARNAME=BROKENWRKLOG ` || fail=yes
|
|
BUILDLOG=`${BMAKE} show-var VARNAME=BUILDLOG` || fail=yes
|
|
STARTFILE=`${BMAKE} show-var VARNAME=STARTFILE` || fail=yes
|
|
AWK=`${BMAKE} show-var VARNAME=AWK` || fail=yes
|
|
GREP=`${BMAKE} show-var VARNAME=GREP` || fail=yes
|
|
SED=`${BMAKE} show-var VARNAME=SED` || fail=yes
|
|
MAIL_CMD=`${BMAKE} show-var VARNAME=MAIL_CMD` || fail=yes
|
|
MACHINE_ARCH=`${BMAKE} show-var VARNAME=MACHINE_ARCH` || fail=yes
|
|
OPSYS=`${BMAKE} show-var VARNAME=OPSYS` || fail=yes
|
|
OS_VERSION=`${BMAKE} show-var VARNAME=OS_VERSION` || fail=yes
|
|
else
|
|
echo "The pkgtools/pkglint directory does not exist. Please update"
|
|
echo "your pkgsrc tree in ${USR_PKGSRC}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "----------------------------------"
|
|
echo "| Build Temporary Files: |"
|
|
echo "----------------------------------"
|
|
echo "BULK_DBFILE = $BULK_DBFILE"
|
|
echo "DEPENDSTREEFILE = $DEPENDSTREEFILE"
|
|
echo "DEPENDSFILE = $DEPENDSFILE"
|
|
echo "SUPPORTSFILE = $SUPPORTSFILE"
|
|
echo "INDEXFILE = $INDEXFILE"
|
|
echo "ORDERFILE = $ORDERFILE"
|
|
echo "BROKENFILE = $BROKENFILE"
|
|
echo "BROKENWRKLOG = $BROKENWRKLOG"
|
|
echo "BUILDLOG = $BUILDLOG"
|
|
echo "STARTFILE = $STARTFILE"
|
|
echo "----------------------------------"
|
|
|
|
# make sure we have values for these very important
|
|
# variables
|
|
if [ $fail = "yes" -o \
|
|
-z "$PERL5_PKGSRCDIR" -o \
|
|
-z "$BULK_DBFILE" -o \
|
|
-z "$DEPENDSTREEFILE" -o \
|
|
-z "$DEPENDSFILE" -o \
|
|
-z "$SUPPORTSFILE" -o \
|
|
-z "$INDEXFILE" -o \
|
|
-z "$ORDERFILE" -o \
|
|
-z "$BROKENFILE" -o \
|
|
-z "$BROKENWRKLOG" -o \
|
|
-z "$BUILDLOG" -o \
|
|
-z "$STARTFILE" -o \
|
|
-z "$AWK" -o \
|
|
-z "$GREP" -o \
|
|
-z "$SED" -o \
|
|
-z "$MAIL_CMD" -o \
|
|
-z "$MACHINE_ARCH" -o \
|
|
-z "$OPSYS" -o \
|
|
-z "$OS_VERSION" \
|
|
]; then
|
|
echo "ERROR: build failed to extract certain key variables."
|
|
echo " please examine the above list and correct the"
|
|
echo " problem."
|
|
exit 1
|
|
fi
|
|
|
|
cd ${USR_PKGSRC}
|
|
|
|
# get the list of packages which should always be installed during the build
|
|
cd ${USR_PKGSRC}/pkgtools/pkglint
|
|
BULK_PREREQ=`${BMAKE} show-var VARNAME=BULK_PREREQ`
|
|
cd ${USR_PKGSRC}
|
|
|
|
# install prerequisite packages. Note: we do this _before_ the depends tree
|
|
# because some packages like xpkgwedge only become DEPENDS if its installed
|
|
echo "Installing prerequisite packages specified with BULK_PREREQ..."
|
|
for pkgdir in $BULK_PREREQ
|
|
do
|
|
echo $pkgdir
|
|
# make sure its installed _and_ packaged
|
|
cd ${USR_PKGSRC}/$pkgdir && ${BMAKE} bulk-install
|
|
done
|
|
|
|
# Create the bulk cache files
|
|
cd ${USR_PKGSRC}
|
|
if [ "x$restart" != "xyes" ]; then
|
|
cd ${USR_PKGSRC} && ${BMAKE} bulk-cache $makeargs
|
|
if [ $? != 0 ]; then
|
|
echo "$0: Cache creation failed. Aborting build."
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "Starting actual build using the order specified in $ORDERFILE..."
|
|
cd ${USR_PKGSRC}
|
|
|
|
# make sure we have something to grep in in the build loop
|
|
touch $BUILDLOG
|
|
|
|
# set the nice level for bulk builds
|
|
#case "${NICE_LEVEL}" in
|
|
#"") NICE_LEVEL="nice -n 20" ;;
|
|
#esac
|
|
|
|
# Loop over every package in the correct order. Before building
|
|
# each one, check to see if we've already processed this package
|
|
# before. This could happen if the build got interrupted and we
|
|
# started it again with the 'restart' option. This prevents us
|
|
# from having to do a potentially very large number of make's to
|
|
# get back to where we let off. After we build each package, add
|
|
# it to the top level buildlog
|
|
# (usually '.make' or '.make.${MACHINE}'). As a side benefit, this
|
|
# can make a progress-meter very simple to add!
|
|
|
|
tot=`wc -l $ORDERFILE | ${AWK} '{print $1}'`
|
|
for pkgdir in `cat $ORDERFILE`
|
|
do
|
|
${GREP} -q "^${pkgdir}\$" $BUILDLOG
|
|
if [ $? -ne 0 ]; then
|
|
built=`wc -l $BUILDLOG | ${AWK} '{print $1}'`
|
|
percent=`echo $built $tot | ${AWK} '{printf("%4.1f%%",$1*100/$2);}'`
|
|
(cd $pkgdir && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes $target \
|
|
$makeargs </dev/null | \
|
|
${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` $built/${tot}=$percent $pkgdir @ ${MACHINE_ARCH}> ;g")
|
|
echo "$pkgdir" >> $BUILDLOG
|
|
fi
|
|
done
|
|
|
|
echo "Build finished. Removing all installed packages left over from build..."
|
|
|
|
for pkgname in `pkg_info -e \*`
|
|
do
|
|
pkg_info -qe $pkgname
|
|
if [ $? -eq 0 ]; then
|
|
pkgdir=`${GREP} " $pkgname " $INDEXFILE | ${AWK} '{print $1}'`
|
|
case "${BULK_PREREQ}" in
|
|
*$pkgdir* )
|
|
echo "Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
|
|
;;
|
|
* )
|
|
echo pkg_delete -r $pkgname
|
|
pkg_delete -r $pkgname
|
|
pkg_info -qe $pkgname
|
|
if [ $? -eq 0 ]; then
|
|
echo "$pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall"
|
|
pkg_delete -f $pkgname || true
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
done
|
|
|
|
# for now, just quit if we were only mirroring distfiles. At somepoint we
|
|
# should teach the post-build script to generate a nice report about how many
|
|
# distfiles were downloaded, how many had bad checksums, failed master sites,
|
|
# network speed, etc.
|
|
|
|
if [ "x$mirror_only" = "xyes" ]; then
|
|
echo "Bulk mirror of distfiles completed: `date`"
|
|
exit 0
|
|
fi
|
|
|
|
echo "Post processing bulk build results..."
|
|
|
|
#rm $DEPENDSTREEFILE $DEPENDSFILE $SUPPORTSFILE $INDEXFILE $ORDERFILE
|
|
|
|
# Perl was wiped, reinstall it!
|
|
( cd pkgtools/pkglint && cd ${PERL5_PKGSRCDIR} && ${BMAKE} bulk-install )
|
|
perl mk/bulk/post-build | ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results `date +%Y-%m-%d`" $ADMIN
|
|
|
|
# Done!
|
|
echo ""
|
|
echo Bulk build ended: `date`
|