cc977aac3f
last file in a directory is removed. This might cause common and expected-to-be-existing directories in /usr/pkg to be removed, and a subsequent attempt at installing a file to the now non-existent directory will instead create a file with the name of the expected directory. This will create PLIST errors for the package in question, but also will cause the erroneously named file to not be removed on package removal. This can cause cascading bulk build failures for subsequent packages. To prevent this, after each package is done, check if some of the common top-level expected-to-be directories are now files, and remove them and emit an error message if so. I *think* I already fixed the single package which had the unfortunate problem of not declaring "include" as an installation directory, but this should prevent the problem from re-occurring in the future as well. Thanks to joerg@ for the hint for pulling in the value of PREFIX.
505 lines
13 KiB
Bash
505 lines
13 KiB
Bash
#!/bin/sh
|
|
# $NetBSD: build,v 1.106 2009/07/09 12:46:49 he Exp $
|
|
|
|
#
|
|
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
# 3. All advertising materials mentioning features or use of this software
|
|
# must display the following acknowledgement:
|
|
# This product includes software developed by Hubert Feyrer for
|
|
# the NetBSD Project.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
##
|
|
## Globals
|
|
##
|
|
scriptdir=`dirname "$0"`
|
|
scriptdir=`cd "${scriptdir}" && pwd`
|
|
|
|
#
|
|
# Default values for command line options.
|
|
#
|
|
resume=no
|
|
mirror_only=no
|
|
target=bulk-package
|
|
makeargs=""
|
|
noemail=no
|
|
post_only=no
|
|
prepare_only=no
|
|
|
|
##
|
|
## Functions
|
|
##
|
|
|
|
usage () {
|
|
cat <<EOF
|
|
usage: $0 [options]
|
|
$0 -h | --help
|
|
|
|
Runs a bulk pkgsrc build.
|
|
|
|
The following options are supported:
|
|
|
|
-c | --config <file>
|
|
Load the following configuration file instead of the default
|
|
one.
|
|
|
|
-e | --no-email
|
|
Don't send email when the bulk build is finished.
|
|
|
|
-h | --help
|
|
Displays this message.
|
|
|
|
-m | --mirror_only
|
|
Downloads all distfiles needed for the build but does not run
|
|
the build. IMPORTANT: Note that this will still run all the
|
|
pre-build stuff which involves removing all of your installed
|
|
packages.
|
|
|
|
The only difference between this option and a regular bulk build
|
|
is that the packages are not actually built.
|
|
|
|
-p | --post-build
|
|
Run the post-build processing and generate the report only.
|
|
|
|
--prepare
|
|
Only generate the package database that will be used when bulk
|
|
building the packages.
|
|
|
|
-r | --resume
|
|
Resume a previously interrupted bulk build.
|
|
|
|
The --resume option may be combined with the --mirror_only
|
|
option.
|
|
|
|
-s | --specific-pkgs
|
|
Sets SPECIFIC_PKGS=1 when building packages. This option is
|
|
used for building a subset of pkgsrc.
|
|
|
|
EOF
|
|
}
|
|
|
|
# print out error message and exit 1
|
|
die () {
|
|
echo "$0: error:" 1>&2
|
|
for i in "$@"; do
|
|
echo " $i" 1>&2
|
|
done
|
|
exit 1
|
|
}
|
|
|
|
# This function can be overridden in the build.conf file to change the
|
|
# output format of the bulk build. It is used in a pipe, so if you want
|
|
# the original output, just define post_filter_cmd() { cat; }.
|
|
#
|
|
# For more sophisticated output, you may use all the variables that this
|
|
# example function uses.
|
|
post_filter_cmd() {
|
|
${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'` ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g"
|
|
}
|
|
|
|
# perform post-processing of the bulk-build results
|
|
do_post_build () {
|
|
echo "build> Post processing bulk build results..."
|
|
|
|
# Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by
|
|
# them for post-build to run.
|
|
echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..."
|
|
for pkgdir in $BULK_PREREQ lang/perl5; do
|
|
echo "build> Installing prerequisite package $pkgdir"
|
|
( cd "${pkgsrc_dir}/${pkgdir}" \
|
|
&& ${BMAKE} bulk-install
|
|
) || die "Failed to install prerequisite packages."
|
|
done
|
|
|
|
#
|
|
# Generate the post-build report.
|
|
#
|
|
echo "build> Generating the bulk build report..."
|
|
|
|
bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
|
|
|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
|
|
|
|
report_dir="${REPORTS_DIR}/${bulk_build_id}"
|
|
${MKDIR} "${report_dir}"
|
|
( cd "${pkgsrc_dir}" \
|
|
&& ${PERL5} mk/bulk/post-build \
|
|
> "${report_dir}/${REPORT_TXT_FILE}"
|
|
) || die "Could not write the results file."
|
|
}
|
|
|
|
# Notify the ADMIN of the finished build.
|
|
do_email () {
|
|
case $noemail in
|
|
no) cat "${report_dir}/${REPORT_TXT_FILE}" \
|
|
| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN"
|
|
esac
|
|
}
|
|
|
|
# output final note that we're done
|
|
do_done () {
|
|
echo ""
|
|
echo "build> Bulk build ended: `date`"
|
|
}
|
|
|
|
# set all commonly used variables, prepare files etc.
|
|
do_common_setup () {
|
|
|
|
#
|
|
# Choose an appropriate value for BMAKE depending on the operating
|
|
# system.
|
|
#
|
|
opsys=`uname -s`
|
|
case "$opsys" in
|
|
NetBSD) BMAKE=make ;;
|
|
*) BMAKE=bmake ;;
|
|
esac
|
|
export BMAKE
|
|
|
|
#
|
|
# Set resource limits as high as possible
|
|
#
|
|
ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack?
|
|
ulimit -S -d `ulimit -H -d` # some processes grow rather large
|
|
ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time
|
|
|
|
#
|
|
# Find the configuration file.
|
|
#
|
|
BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
|
|
case $BULK_BUILD_CONF in
|
|
/*) ;;
|
|
*) BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
|
|
esac
|
|
|
|
#
|
|
# Load the variables from the configuration file.
|
|
#
|
|
{ test -f "${BULK_BUILD_CONF}" \
|
|
&& . "${BULK_BUILD_CONF}" \
|
|
&& . "${scriptdir}/post-build-conf" \
|
|
&& check_config_vars \
|
|
&& export_config_vars
|
|
} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."
|
|
|
|
#
|
|
# Set the paths to commonly used directories.
|
|
#
|
|
pkgsrc_dir="${USR_PKGSRC}"
|
|
lintpkgsrc_dir="${USR_PKGSRC}/pkgtools/lintpkgsrc"
|
|
|
|
#
|
|
# Set up variables specific for the bulk build.
|
|
#
|
|
BATCH="1"
|
|
DEPENDS_TARGET="bulk-install"
|
|
export BATCH DEPENDS_TARGET
|
|
|
|
#
|
|
# Unset some environment variables that could disturbe the build.
|
|
#
|
|
unset CDPATH || true # ensure cd does not print new cwd to stdout, which
|
|
# confuses the printindex script.
|
|
unset DISPLAY || true # allow sane failure for gimp, xlispstat
|
|
}
|
|
|
|
# Check that the package tools are up to date.
|
|
check_tools () {
|
|
echo "build> Checking if the pkgtools are up-to-date"
|
|
( cd "${lintpkgsrc_dir}" \
|
|
&& ${BMAKE} fetch >/dev/null 2>&1
|
|
) || {
|
|
echo "build> Updating pkgtools"
|
|
( cd "${pkgsrc_dir}/pkgtools/pkg_install" \
|
|
&& ${BMAKE} clean \
|
|
&& ${BMAKE} update
|
|
) || die "Could not update the package tools."
|
|
}
|
|
}
|
|
|
|
# Run the pre-build script if necessary.
|
|
run_pre_build () {
|
|
case $resume in
|
|
yes) echo "build> Resuming -- skipping pre-build script";;
|
|
*) # make veryveryclean :)
|
|
( cd "${pkgsrc_dir}" \
|
|
&& /bin/sh mk/bulk/pre-build
|
|
) || die "Error during bulk-build preparations, aborting.";;
|
|
esac
|
|
}
|
|
|
|
# Load pkgsrc variables that affect the build process.
|
|
load_vars () {
|
|
echo "+----------------------------------------+"
|
|
echo "| Some variables used in the bulk build: |"
|
|
echo "+----------------------------------------+"
|
|
|
|
vars=" OPSYS OS_VERSION MACHINE_ARCH
|
|
BULK_PREREQ
|
|
BULKFILESDIR
|
|
BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE
|
|
SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE
|
|
BROKENWRKLOG
|
|
PREFIX
|
|
AWK GREP MAIL_CMD MKDIR PAX PERL5 SED
|
|
PKG_DELETE PKG_INFO PKGBASE"
|
|
|
|
values=`cd "$lintpkgsrc_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir pax perl sed"`
|
|
|
|
for v in $vars; do
|
|
eval "read $v" || die "Could not read value for $v"
|
|
eval "value=\$$v"
|
|
if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then
|
|
die "$v must not be empty."
|
|
fi
|
|
printf "%-15s = %s\\n" "$v" "$value"
|
|
done <<EOF
|
|
$values
|
|
EOF
|
|
echo "------------------------------------------"
|
|
[ "$PKGBASE" = "lintpkgsrc" ] \
|
|
|| die "Error reading the variables." \
|
|
"Try running $0 with another shell."
|
|
|
|
# Get the location of commonly used files
|
|
main_buildlog="${BULKFILESDIR}/${BUILDLOG}"
|
|
}
|
|
|
|
# Install prerequisite packages.
|
|
#
|
|
# Note: we do this _before_ the depends tree because some packages like
|
|
# xpkgwedge only become DEPENDS if they are installed.
|
|
install_prereqs () {
|
|
echo "build> Installing prerequisite packages specified with BULK_PREREQ..."
|
|
for pkgdir in $BULK_PREREQ; do
|
|
echo "build> Installing prerequisite package $pkgdir"
|
|
( cd "${pkgsrc_dir}/${pkgdir}" \
|
|
&& ${BMAKE} bulk-install
|
|
) || die "Could not install prerequisite packages."
|
|
done
|
|
}
|
|
|
|
# Everything is prepared. We can start building the real packages now.
|
|
#
|
|
# Loop over every package in the correct order. Before building
|
|
# each one, check to see if we've already processed this package
|
|
# before. This could happen if the build got interrupted and we
|
|
# started it again with the '-resume' option. This prevents us
|
|
# from having to do a potentially very large number of make's to
|
|
# get back to where we let off. After we build each package, add
|
|
# it to the top level buildlog
|
|
# (usually '.make' or '.make.${MACHINE}'). As a side benefit, this
|
|
# can make a progress-meter very simple to add!
|
|
do_real_bulk_build () {
|
|
|
|
cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist."
|
|
|
|
echo "build> Starting actual build using the order specified in $ORDERFILE..."
|
|
|
|
# make sure we have something to grep in in the build loop
|
|
touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}."
|
|
|
|
tot=`${AWK} 'END { print NR }' "${ORDERFILE}"`
|
|
for pkgdir in `cat "${ORDERFILE}"`
|
|
do
|
|
if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then
|
|
: "skip this package"
|
|
else
|
|
percent=`${AWK} -v tot="${tot}" '
|
|
END {
|
|
printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot);
|
|
}' "${main_buildlog}"`
|
|
( cd "${pkgsrc_dir}/${pkgdir}" \
|
|
&& ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \
|
|
$makeargs </dev/null | post_filter_cmd
|
|
) || true
|
|
echo "$pkgdir" >> "${main_buildlog}"
|
|
check_pkg_dirs
|
|
fi
|
|
done
|
|
|
|
echo "build> Build finished."
|
|
}
|
|
|
|
check_pkg_dirs () {
|
|
for d in bin etc include info lib libexec man sbin share; do
|
|
if [ -f $PREFIX/$d ]; then
|
|
echo "Removing file, should be dir: $PREFIX/$d" >&2
|
|
rm -f $PREFIX/$d
|
|
fi
|
|
done
|
|
}
|
|
|
|
# clean up installed packages left over
|
|
do_bulk_cleanup () {
|
|
|
|
echo "build> Removing all installed packages left over from build..."
|
|
for pkgname in `${PKG_INFO} -e \*`
|
|
do
|
|
if ${PKG_INFO} -qe "${pkgname}"; then
|
|
pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"`
|
|
case "${BULK_PREREQ}" in
|
|
*"${pkgdir}"* )
|
|
echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
|
|
;;
|
|
* )
|
|
echo "build> ${PKG_DELETE} -r ${pkgname}"
|
|
${PKG_DELETE} -r "${pkgname}"
|
|
if ${PKG_INFO} -qe "${pkgname}"; then
|
|
echo "build> $pkgname ($pkgdir) did not deinstall nicely. Forcing the deinstall"
|
|
${PKG_DELETE} -f "${pkgname}" || true
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
done
|
|
}
|
|
|
|
# start the full bulk-build
|
|
do_bulk_build () {
|
|
echo "build> Bulk build started: `date`"
|
|
echo ""
|
|
|
|
# this function from post-build-conf
|
|
show_config_vars
|
|
|
|
check_tools
|
|
run_pre_build
|
|
load_vars
|
|
|
|
#
|
|
# Create the directory for the log files if necessary
|
|
#
|
|
if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
|
|
${MKDIR} "${BULKFILESDIR}"
|
|
fi
|
|
|
|
#
|
|
# Save the bulk build ID in a file, as it most often contains a time
|
|
# stamp.
|
|
#
|
|
case $resume in
|
|
no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
|
|
|| die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
|
|
esac
|
|
|
|
install_prereqs
|
|
|
|
#
|
|
# Create the bulk cache files.
|
|
#
|
|
if [ "x$resume" != "xyes" ]; then
|
|
( cd "${pkgsrc_dir}" \
|
|
&& env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs
|
|
) || die "Could not create the bulk build cache."
|
|
else
|
|
if [ ! -f "${ORDERFILE}" ]; then
|
|
die "The ${ORDERFILE} does not exist." \
|
|
"(You cannot resume a bulk build that has not yet started.)"
|
|
fi
|
|
fi
|
|
|
|
# XXX: This looks like a hack, and indeed, the functions in this file
|
|
# should be reorganized to better reflect the phases of the bulk build.
|
|
if [ $prepare_only = yes ]; then
|
|
exit 0
|
|
fi
|
|
|
|
do_real_bulk_build
|
|
do_bulk_cleanup
|
|
}
|
|
|
|
|
|
##
|
|
## main
|
|
##
|
|
|
|
#
|
|
# Parse the command line.
|
|
#
|
|
while test $# -gt 0; do
|
|
case $1 in
|
|
-c|--config)
|
|
shift
|
|
BULK_BUILD_CONF=$1; shift
|
|
;;
|
|
-e|--no-email)
|
|
noemail=yes
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
-m|--mirror_only)
|
|
mirror_only=yes
|
|
target=mirror-distfiles
|
|
shift
|
|
;;
|
|
-p|--post-build)
|
|
post_only=yes
|
|
shift
|
|
;;
|
|
--prepare)
|
|
prepare_only=yes
|
|
shift
|
|
;;
|
|
-r|--resume|--restart|restart)
|
|
resume=yes
|
|
shift
|
|
;;
|
|
-s|--specific-pkgs)
|
|
makeargs="$makeargs SPECIFIC_PKGS=1"
|
|
shift
|
|
;;
|
|
*)
|
|
echo "unknown option: $1" 1>&2
|
|
usage 1>&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
do_common_setup
|
|
|
|
if [ "x$post_only" = "xyes" ]; then
|
|
load_vars
|
|
do_post_build
|
|
exit 0
|
|
fi
|
|
|
|
do_bulk_build
|
|
|
|
# for now, just quit if we were only mirroring distfiles. At somepoint we
|
|
# should teach the post-build script to generate a nice report about how many
|
|
# distfiles were downloaded, how many had bad checksums, failed master sites,
|
|
# network speed, etc.
|
|
if [ "x$mirror_only" = "xyes" ]; then
|
|
echo "build> Bulk mirror of distfiles completed: `date`"
|
|
exit 0
|
|
fi
|
|
|
|
do_post_build
|
|
do_email
|
|
do_done
|
|
|