495 lines
13 KiB
Bash
495 lines
13 KiB
Bash
#!/bin/sh
|
|
# $NetBSD: build,v 1.103 2008/01/04 15:49:07 rillig Exp $
|
|
|
|
#
|
|
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
# 3. All advertising materials mentioning features or use of this software
|
|
# must display the following acknowledgement:
|
|
# This product includes software developed by Hubert Feyrer for
|
|
# the NetBSD Project.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
##
|
|
## Globals
|
|
##
|
|
scriptdir=`dirname "$0"`
|
|
scriptdir=`cd "${scriptdir}" && pwd`
|
|
|
|
#
|
|
# Default values for command line options.
|
|
#
|
|
resume=no
|
|
mirror_only=no
|
|
target=bulk-package
|
|
makeargs=""
|
|
noemail=no
|
|
post_only=no
|
|
prepare_only=no
|
|
|
|
##
|
|
## Functions
|
|
##
|
|
|
|
usage () {
|
|
cat <<EOF
|
|
usage: $0 [options]
|
|
$0 -h | --help
|
|
|
|
Runs a bulk pkgsrc build.
|
|
|
|
The following options are supported:
|
|
|
|
-c | --config <file>
|
|
Load the following configuration file instead of the default
|
|
one.
|
|
|
|
-e | --no-email
|
|
Don't send email when the bulk build is finished.
|
|
|
|
-h | --help
|
|
Displays this message.
|
|
|
|
-m | --mirror_only
|
|
Downloads all distfiles needed for the build but does not run
|
|
the build. IMPORTANT: Note that this will still run all the
|
|
pre-build stuff which involves removing all of your installed
|
|
packages.
|
|
|
|
The only difference between this option and a regular bulk build
|
|
is that the packages are not actually built.
|
|
|
|
-p | --post-build
|
|
Run the post-build processing and generate the report only.
|
|
|
|
--prepare
|
|
Only generate the package database that will be used when bulk
|
|
building the packages.
|
|
|
|
-r | --resume
|
|
Resume a previously interrupted bulk build.
|
|
|
|
The --resume option may be combined with the --mirror_only
|
|
option.
|
|
|
|
-s | --specific-pkgs
|
|
Sets SPECIFIC_PKGS=1 when building packages. This option is
|
|
used for building a subset of pkgsrc.
|
|
|
|
EOF
|
|
}
|
|
|
|
# print out error message and exit 1
|
|
die () {
|
|
echo "$0: error:" 1>&2
|
|
for i in "$@"; do
|
|
echo " $i" 1>&2
|
|
done
|
|
exit 1
|
|
}
|
|
|
|
# This function can be overridden in the build.conf file to change the
|
|
# output format of the bulk build. It is used in a pipe, so if you want
|
|
# the original output, just define post_filter_cmd() { cat; }.
|
|
#
|
|
# For more sophisticated output, you may use all the variables that this
|
|
# example function uses.
|
|
post_filter_cmd() {
|
|
${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g"
|
|
}
|
|
|
|
# perform post-processing of the bulk-build results
|
|
do_post_build () {
|
|
echo "build> Post processing bulk build results..."
|
|
|
|
# Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by
|
|
# them for post-build to run.
|
|
echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..."
|
|
for pkgdir in $BULK_PREREQ lang/perl5; do
|
|
echo "build> Installing prerequisite package $pkgdir"
|
|
( cd "${pkgsrc_dir}/${pkgdir}" \
|
|
&& ${BMAKE} bulk-install
|
|
) || die "Failed to install prerequisite packages."
|
|
done
|
|
|
|
#
|
|
# Generate the post-build report.
|
|
#
|
|
echo "build> Generating the bulk build report..."
|
|
|
|
bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
|
|
|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
|
|
|
|
report_dir="${REPORTS_DIR}/${bulk_build_id}"
|
|
${MKDIR} "${report_dir}"
|
|
( cd "${pkgsrc_dir}" \
|
|
&& ${PERL5} mk/bulk/post-build \
|
|
> "${report_dir}/${REPORT_TXT_FILE}"
|
|
) || die "Could not write the results file."
|
|
}
|
|
|
|
# Notify the ADMIN of the finished build.
|
|
do_email () {
|
|
case $noemail in
|
|
no) cat "${report_dir}/${REPORT_TXT_FILE}" \
|
|
| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN"
|
|
esac
|
|
}
|
|
|
|
# output final note that we're done
|
|
do_done () {
|
|
echo ""
|
|
echo "build> Bulk build ended: `date`"
|
|
}
|
|
|
|
# set all commonly used variables, prepare files etc.
|
|
do_common_setup () {
|
|
|
|
#
|
|
# Choose an appropriate value for BMAKE depending on the operating
|
|
# system.
|
|
#
|
|
opsys=`uname -s`
|
|
case "$opsys" in
|
|
NetBSD) BMAKE=make ;;
|
|
*) BMAKE=bmake ;;
|
|
esac
|
|
export BMAKE
|
|
|
|
#
|
|
# Set resource limits as high as possible
|
|
#
|
|
ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack?
|
|
ulimit -S -d `ulimit -H -d` # some processes grow rather large
|
|
ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time
|
|
|
|
#
|
|
# Find the configuration file.
|
|
#
|
|
BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
|
|
case $BULK_BUILD_CONF in
|
|
/*) ;;
|
|
*) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
|
|
esac
|
|
|
|
#
|
|
# Load the variables from the configuration file.
|
|
#
|
|
{ test -f "${BULK_BUILD_CONF}" \
|
|
&& . "${BULK_BUILD_CONF}" \
|
|
&& . "${scriptdir}/post-build-conf" \
|
|
&& check_config_vars \
|
|
&& export_config_vars
|
|
} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."
|
|
|
|
#
|
|
# Set the paths to commonly used directories.
|
|
#
|
|
pkgsrc_dir="${USR_PKGSRC}"
|
|
lintpkgsrc_dir="${USR_PKGSRC}/pkgtools/lintpkgsrc"
|
|
|
|
#
|
|
# Set up variables specific for the bulk build.
|
|
#
|
|
BATCH="1"
|
|
DEPENDS_TARGET="bulk-install"
|
|
export BATCH DEPENDS_TARGET
|
|
|
|
#
|
|
# Unset some environment variables that could disturbe the build.
|
|
#
|
|
unset CDPATH || true # ensure cd does not print new cwd to stdout, which
|
|
# confuses the printindex script.
|
|
unset DISPLAY || true # allow sane failure for gimp, xlispstat
|
|
}
|
|
|
|
# Check that the package tools are up to date.
|
|
check_tools () {
|
|
echo "build> Checking if the pkgtools are up-to-date"
|
|
( cd "${lintpkgsrc_dir}" \
|
|
&& ${BMAKE} fetch >/dev/null 2>&1
|
|
) || {
|
|
echo "build> Updating pkgtools"
|
|
( cd "${pkgsrc_dir}/pkgtools/pkg_install" \
|
|
&& ${BMAKE} clean \
|
|
&& ${BMAKE} install \
|
|
&& ${BMAKE} clean
|
|
) || die "Could not update the package tools."
|
|
}
|
|
}
|
|
|
|
# Run the pre-build script if necessary.
|
|
run_pre_build () {
|
|
case $resume in
|
|
yes) echo "build> Resuming -- skipping pre-build script";;
|
|
*) # make veryveryclean :)
|
|
( cd "${pkgsrc_dir}" \
|
|
&& /bin/sh mk/bulk/pre-build
|
|
) || die "Error during bulk-build preparations, aborting.";;
|
|
esac
|
|
}
|
|
|
|
# Load pkgsrc variables that affect the build process.
|
|
load_vars () {
|
|
echo "+----------------------------------------+"
|
|
echo "| Some variables used in the bulk build: |"
|
|
echo "+----------------------------------------+"
|
|
|
|
vars=" OPSYS OS_VERSION MACHINE_ARCH
|
|
BULK_PREREQ
|
|
BULKFILESDIR
|
|
BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE
|
|
SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE
|
|
BROKENWRKLOG
|
|
AWK GREP MAIL_CMD MKDIR PERL5 SED
|
|
PKG_DELETE PKG_INFO PKGBASE"
|
|
|
|
values=`cd "$lintpkgsrc_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir perl sed"`
|
|
|
|
for v in $vars; do
|
|
eval "read $v" || die "Could not read value for $v"
|
|
eval "value=\$$v"
|
|
if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then
|
|
die "$v must not be empty."
|
|
fi
|
|
printf "%-15s = %s\\n" "$v" "$value"
|
|
done <<EOF
|
|
$values
|
|
EOF
|
|
echo "------------------------------------------"
|
|
[ "$PKGBASE" = "lintpkgsrc" ] \
|
|
|| die "Error reading the variables." \
|
|
"Try running $0 with another shell."
|
|
|
|
# Get the location of commonly used files
|
|
main_buildlog="${BULKFILESDIR}/${BUILDLOG}"
|
|
}
|
|
|
|
# Install prerequisite packages.
|
|
#
|
|
# Note: we do this _before_ the depends tree because some packages like
|
|
# xpkgwedge only become DEPENDS if they are installed.
|
|
install_prereqs () {
|
|
echo "build> Installing prerequisite packages specified with BULK_PREREQ..."
|
|
for pkgdir in $BULK_PREREQ; do
|
|
echo "build> Installing prerequisite package $pkgdir"
|
|
( cd "${pkgsrc_dir}/${pkgdir}" \
|
|
&& ${BMAKE} bulk-install
|
|
) || die "Could not install prerequisite packages."
|
|
done
|
|
}
|
|
|
|
# Everything is prepared. We can start building the real packages now.
|
|
#
|
|
# Loop over every package in the correct order. Before building
|
|
# each one, check to see if we've already processed this package
|
|
# before. This could happen if the build got interrupted and we
|
|
# started it again with the '-resume' option. This prevents us
|
|
# from having to do a potentially very large number of make's to
|
|
# get back to where we let off. After we build each package, add
|
|
# it to the top level buildlog
|
|
# (usually '.make' or '.make.${MACHINE}'). As a side benefit, this
|
|
# can make a progress-meter very simple to add!
|
|
do_real_bulk_build () {
|
|
|
|
cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist."
|
|
|
|
echo "build> Starting actual build using the order specified in $ORDERFILE..."
|
|
|
|
# make sure we have something to grep in in the build loop
|
|
touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}."
|
|
|
|
tot=`${AWK} 'END { print NR }' "${ORDERFILE}"`
|
|
for pkgdir in `cat "${ORDERFILE}"`
|
|
do
|
|
if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then
|
|
: "skip this package"
|
|
else
|
|
percent=`${AWK} -v tot="${tot}" '
|
|
END {
|
|
printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot);
|
|
}' "${main_buildlog}"`
|
|
( cd "${pkgsrc_dir}/${pkgdir}" \
|
|
&& ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \
|
|
$makeargs </dev/null | post_filter_cmd
|
|
) || true
|
|
echo "$pkgdir" >> "${main_buildlog}"
|
|
fi
|
|
done
|
|
|
|
echo "build> Build finished."
|
|
}
|
|
|
|
# clean up installed packages left over
|
|
do_bulk_cleanup () {
|
|
|
|
echo "build> Removing all installed packages left over from build..."
|
|
for pkgname in `${PKG_INFO} -e \*`
|
|
do
|
|
if ${PKG_INFO} -qe "${pkgname}"; then
|
|
pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"`
|
|
case "${BULK_PREREQ}" in
|
|
*"${pkgdir}"* )
|
|
echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
|
|
;;
|
|
* )
|
|
echo "build> ${PKG_DELETE} -r ${pkgname}"
|
|
${PKG_DELETE} -r "${pkgname}"
|
|
if ${PKG_INFO} -qe "${pkgname}"; then
|
|
echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall"
|
|
${PKG_DELETE} -f "${pkgname}" || true
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
done
|
|
}
|
|
|
|
# start the full bulk-build
|
|
do_bulk_build () {
|
|
echo "build> Bulk build started: `date`"
|
|
echo ""
|
|
|
|
# this function from post-build-conf
|
|
show_config_vars
|
|
|
|
check_tools
|
|
run_pre_build
|
|
load_vars
|
|
|
|
#
|
|
# Create the directory for the log files if necessary
|
|
#
|
|
if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
|
|
${MKDIR} "${BULKFILESDIR}"
|
|
fi
|
|
|
|
#
|
|
# Save the bulk build ID in a file, as it most often contains a time
|
|
# stamp.
|
|
#
|
|
case $resume in
|
|
no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
|
|
|| die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
|
|
esac
|
|
|
|
install_prereqs
|
|
|
|
#
|
|
# Create the bulk cache files.
|
|
#
|
|
if [ "x$resume" != "xyes" ]; then
|
|
( cd "${pkgsrc_dir}" \
|
|
&& env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs
|
|
) || die "Could not create the bulk build cache."
|
|
else
|
|
if [ ! -f "${ORDERFILE}" ]; then
|
|
die "The ${ORDERFILE} does not exist." \
|
|
"(You cannot resume a bulk build that has not yet started.)"
|
|
fi
|
|
fi
|
|
|
|
# XXX: This looks like a hack, and indeed, the functions in this file
|
|
# should be reorganized to better reflect the phases of the bulk build.
|
|
if [ $prepare_only = yes ]; then
|
|
exit 0
|
|
fi
|
|
|
|
do_real_bulk_build
|
|
do_bulk_cleanup
|
|
}
|
|
|
|
|
|
##
|
|
## main
|
|
##
|
|
|
|
#
|
|
# Parse the command line.
|
|
#
|
|
while test $# -gt 0; do
|
|
case $1 in
|
|
-c|--config)
|
|
shift
|
|
BULK_BUILD_CONF=$1; shift
|
|
;;
|
|
-e|--no-email)
|
|
noemail=yes
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
-m|--mirror_only)
|
|
mirror_only=yes
|
|
target=mirror-distfiles
|
|
shift
|
|
;;
|
|
-p|--post-build)
|
|
post_only=yes
|
|
shift
|
|
;;
|
|
--prepare)
|
|
prepare_only=yes
|
|
shift
|
|
;;
|
|
-r|--resume|--restart|restart)
|
|
resume=yes
|
|
shift
|
|
;;
|
|
-s|--specific-pkgs)
|
|
makeargs="$makeargs SPECIFIC_PKGS=1"
|
|
shift
|
|
;;
|
|
*)
|
|
echo "unknown option: $1" 1>&2
|
|
usage 1>&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
do_common_setup
|
|
|
|
if [ "x$post_only" = "xyes" ]; then
|
|
load_vars
|
|
do_post_build
|
|
exit 0
|
|
fi
|
|
|
|
do_bulk_build
|
|
|
|
# for now, just quit if we were only mirroring distfiles. At somepoint we
|
|
# should teach the post-build script to generate a nice report about how many
|
|
# distfiles were downloaded, how many had bad checksums, failed master sites,
|
|
# network speed, etc.
|
|
if [ "x$mirror_only" = "xyes" ]; then
|
|
echo "build> Bulk mirror of distfiles completed: `date`"
|
|
exit 0
|
|
fi
|
|
|
|
do_post_build
|
|
do_email
|
|
do_done
|
|
|