pkgsrc/mk/bulk/build

#!/bin/sh
# $NetBSD: build,v 1.87 2005/11/28 21:11:36 rillig Exp $

#
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Hubert Feyrer for
#	the NetBSD Project.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

#
# Do bulk build
#

usage() {
	cat <<EOF
usage: $0 [options]
       $0 -h | --help

Runs a bulk pkgsrc build.

The following options are supported:

   -c | --config <file>
	Load the following configuration file instead of the default
	one.

   -e | --no-email
	Don't send email when the bulk build is finished, it will put
	the results into a file (FTP/pkgsrc-results.txt).

   -h | --help
	Displays this message.

   -m | --mirror_only
	Downloads all distfiles needed for the build but does not run
	the build.  IMPORTANT:  Note that this will still run all the
	pre-build stuff which involves removing all of your installed
	packages.

	The only difference between this option and a regular bulk build
	is that the packages are not actually built.

   -r | --restart | --resume | restart
	Restart a previously interrupted bulk build.  The last form of
	this option is for backwards compatibility and may be removed in
	future versions of this script.

	The --restart option may be combined with the --mirror_only
	option.

   -s | --specific-pkgs
	Sets SPECIFIC_PKGS=1 when building packages.  This option is
	used for building a subset of pkgsrc.

EOF
}

die() {
	echo "$0: error:" 1>&2
	for i in "$@"; do
		echo "    $i" 1>&2
	done
	exit 1
}

# This function can be overridden in the build.conf file to change the
# output format of the bulk build. It is used in a pipe, so if you want
# the original output, just define post_filter_cmd() { cat; }.
#
# For more sophisticated output, you may use all the variables that this
# example function uses.
post_filter_cmd() {
	${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'`  ${built}/${tot}=${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g"
}


#
# Find out where we are
#
scriptdir=`dirname "$0"`
scriptdir=`cd "${scriptdir}" && pwd`

#
# Default values for command line options.
#
restart=no
mirror_only=no
target=bulk-package
makeargs=""
noemail=no

#
# Parse the command line.
#
while test $# -gt 0; do
	case $1 in
	-c|--config)
		shift
		BULK_BUILD_CONF=$1; shift
		;;
	-e|--no-email)
		noemail=yes
		shift
		;;
	-h|--help)
		usage
		exit 0
		;;
	-m|--mirror_only)
		mirror_only=yes
		target=mirror-distfiles
		shift
		;;
	-r|--restart|--resume|restart)
		restart=yes
		shift
		;;
	-s|--specific-pkgs)
		makeargs="$makeargs SPECIFIC_PKGS=1"
		shift
		;;
	*)
		echo "unknown option:  $1" 1>&2
		usage 1>&2
		exit 1
		;;
	esac
done

#
# Choose an appropriate value for BMAKE depending on the operating system.
#
opsys=`uname -s`
case "$opsys" in
NetBSD)	BMAKE=make ;;
*)	BMAKE=bmake ;;
esac
export BMAKE

#
# Set resource limits as high as possible
#
ulimit -S -s `ulimit -H -s`
ulimit -S -d `ulimit -H -d`

#
# Find the configuration file.
#
BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
case $BULK_BUILD_CONF in
/*) ;;
*)	BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
esac

#
# Load the variables from the configuration file.
#
{ test -f "${BULK_BUILD_CONF}" \
  && . "${BULK_BUILD_CONF}" \
  && . "${scriptdir}/post-build-conf" \
  && check_config_vars \
  && export_config_vars
} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."

#
# Set the paths to commonly used directories.
#
pkgsrc_dir="${USR_PKGSRC}"
pkglint_dir="${USR_PKGSRC}/pkgtools/pkglint"

#
# Set up variables specific for the bulk build.
#
BATCH="1"
DEPENDS_TARGET="bulk-install"
export BATCH DEPENDS_TARGET

#
# Unset some environment variables that could disturbe the build.
#
unset CDPATH		# ensure cd does not print new cwd to stdout, which
			# confuses the printindex script.
unset DISPLAY		# allow sane failure for gimp, xlispstat

#
# It starts ...
#
echo "build> Bulk build started: `date`"
echo ""
show_config_vars

#
# Check that the package tools are up to date.
#
( cd "${pkglint_dir}" \
  && ${BMAKE} fetch >/dev/null 2>&1
) || {
	echo "build> Updating pkgtools"
	( cd "${pkgsrc_dir}/pkgtools/pkg_install" \
	  && ${BMAKE} clean \
	  && ${BMAKE} install \
	  && ${BMAKE} clean
	) || die "Could not update the package tools."
}

#
# Run the pre-build script if necessary.
#
case $restart in
yes)	echo "build> Resuming -- skipping pre-build script";;
*)	# make veryveryclean :)
	( cd "${pkgsrc_dir}" \
	  && /bin/sh mk/bulk/pre-build
	) || die "Error during bulk-build preparations, aborting.";;
esac

#
# Load pkgsrc variables that affect the build process.
#
fail=no
if cd "${pkglint_dir}"; then
	echo "build> Extracting relevant pkgsrc variables..."

	BULKFILESDIR=`${BMAKE} show-var VARNAME=BULKFILESDIR` || fail=yes
	BULK_DBFILE=`${BMAKE} show-var VARNAME=BULK_DBFILE` || fail=yes
	DEPENDSTREEFILE=`${BMAKE} show-var VARNAME=DEPENDSTREEFILE` || fail=yes
	DEPENDSFILE=`${BMAKE} show-var VARNAME=DEPENDSFILE` || fail=yes
	SUPPORTSFILE=`${BMAKE} show-var VARNAME=SUPPORTSFILE` || fail=yes
	INDEXFILE=`${BMAKE} show-var VARNAME=INDEXFILE` || fail=yes
	ORDERFILE=`${BMAKE} show-var VARNAME=ORDERFILE` || fail=yes
	BROKENFILE=`${BMAKE} show-var VARNAME=BROKENFILE` || fail=yes
	BROKENWRKLOG=`${BMAKE} show-var VARNAME=BROKENWRKLOG ` || fail=yes
	BUILDLOG=`${BMAKE} show-var VARNAME=BUILDLOG` || fail=yes
	STARTFILE=`${BMAKE} show-var VARNAME=STARTFILE` || fail=yes
	AWK=`${BMAKE} show-var VARNAME=AWK USE_TOOLS=awk` || fail=yes
	GREP=`${BMAKE} show-var VARNAME=GREP USE_TOOLS=grep` || fail=yes
	SED=`${BMAKE} show-var VARNAME=SED USE_TOOLS=sed` || fail=yes
	MAIL_CMD=`${BMAKE} show-var VARNAME=MAIL_CMD USE_TOOLS=mail` || fail=yes
	MKDIR=`${BMAKE} show-var VARNAME=MKDIR USE_TOOLS=mkdir` || fail=yes
	PERL5=`${BMAKE} show-var VARNAME=PERL5 USE_TOOLS=perl` || fail=yes
	MACHINE_ARCH=`${BMAKE} show-var VARNAME=MACHINE_ARCH` || fail=yes
	OPSYS=`${BMAKE} show-var VARNAME=OPSYS` || fail=yes
	OS_VERSION=`${BMAKE} show-var VARNAME=OS_VERSION` || fail=yes
	PKG_TOOLS_BIN=`${BMAKE} show-var VARNAME=PKG_TOOLS_BIN` || fail=yes
	BULK_PREREQ=`${BMAKE} show-var VARNAME=BULK_PREREQ` || fail=yes
else
	die	"The pkgtools/pkglint directory does not exist." \
		"Please update your pkgsrc tree in ${pkgsrc_dir}."
fi

echo "+----------------------------------------+"
echo "| Some variables used in the bulk build: |"
echo "+----------------------------------------+"
echo "BULKFILESDIR    = $BULKFILESDIR"
echo "BULK_DBFILE     = $BULK_DBFILE"
echo "BULK_PREREQ     = $BULK_PREREQ"
echo "DEPENDSTREEFILE = $DEPENDSTREEFILE"
echo "DEPENDSFILE     = $DEPENDSFILE"
echo "SUPPORTSFILE    = $SUPPORTSFILE"
echo "INDEXFILE       = $INDEXFILE"
echo "ORDERFILE       = $ORDERFILE"
echo "BROKENFILE      = $BROKENFILE"
echo "BROKENWRKLOG    = $BROKENWRKLOG"
echo "BUILDLOG        = $BUILDLOG"
echo "STARTFILE       = $STARTFILE"
echo "AWK             = $AWK"
echo "GREP            = $GREP"
echo "MKDIR           = $MKDIR"
echo "SED             = $SED"
echo "MAIL_CMD        = $MAIL_CMD"
echo "PERL5           = $PERL5"
echo "MACHINE_ARCH    = $MACHINE_ARCH"
echo "OPSYS           = $OPSYS"
echo "OS_VERSION      = $OS_VERSION"
echo "PKG_TOOLS_BIN   = $PKG_TOOLS_BIN"
echo "------------------------------------------"

# make sure we have values for these very important
# variables; BULK_PREREQ may be empty.
if [ $fail = "yes" -o \
	-z "$BULK_DBFILE" -o \
	-z "$DEPENDSTREEFILE" -o \
	-z "$DEPENDSFILE" -o \
	-z "$SUPPORTSFILE" -o \
	-z "$INDEXFILE" -o \
	-z "$ORDERFILE" -o \
	-z "$BROKENFILE" -o \
	-z "$BROKENWRKLOG" -o \
	-z "$BUILDLOG" -o \
	-z "$STARTFILE" -o \
	-z "$AWK" -o \
	-z "$GREP" -o \
	-z "$MKDIR" -o \
	-z "$SED" -o \
	-z "$MAIL_CMD" -o \
	-z "$PERL5" -o \
	-z "$MACHINE_ARCH" -o \
	-z "$OPSYS" -o \
	-z "$PKG_TOOLS_BIN" -o \
	-z "$OS_VERSION" \
	]; then

	die	"Failed to extract certain key variables." \
		"Please examine the above list and correct the problem."
fi

#
# Get the location of commonly used files
#
main_buildlog="${BULKFILESDIR}/${BUILDLOG}"

#
# Create the directory for the log files if necessary
#
if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
	${MKDIR} "${BULKFILESDIR}"
fi

#
# Install prerequisite packages.
#
# Note: we do this _before_ the depends tree because some packages like
# xpkgwedge only become DEPENDS if they are installed.
#
echo "build> Installing prerequisite packages specified with BULK_PREREQ..."
for pkgdir in $BULK_PREREQ; do
	echo "===> Installing prerequisite package $pkgdir"
	( cd "${pkgsrc_dir}/${pkgdir}" \
	  && ${BMAKE} bulk-install
	) || die "Could not install prerequisite packages."
done

#
# Create the bulk cache files.
#
if [ "x$restart" != "xyes" ]; then
	( cd "${pkgsrc_dir}" \
	  && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs
	) || die "Could not create the bulk build cache."
else
	if [ ! -f "${ORDERFILE}" ]; then
		die	"The ${ORDERFILE} does not exist." \
			"(You cannot resume a bulk build that has not yet started.)"
	fi
fi

#
# Everything is prepared. We can start building the real packages now.
#
cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist."

echo "build> Starting actual build using the order specified in $ORDERFILE..."

# Loop over every package in the correct order.  Before building
# each one, check to see if we've already processed this package
# before.  This could happen if the build got interrupted and we
# started it again with the 'restart' option.  This prevents us
# from having to do a potentially very large number of make's to
# get back to where we let off.  After we build each package, add
# it to the top level buildlog
# (usually '.make' or '.make.${MACHINE}').  As a side benefit, this
# can make a progress-meter very simple to add!

# make sure we have something to grep in in the build loop
touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}."

tot=`wc -l "${ORDERFILE}" | ${AWK} '{print $1}'`
for pkgdir in `cat "${ORDERFILE}"`
do
	if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then
		: "skip this package"
	else
		built=`wc -l "${main_buildlog}" | ${AWK} '{print $1}'`
		percent=`echo $built $tot | ${AWK} '{printf("%4.1f%%",$1*100/$2);}'`
		( cd "${pkgsrc_dir}/${pkgdir}" \
		  && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \
			$makeargs </dev/null | post_filter_cmd
		) || true
		echo "$pkgdir" >> "${main_buildlog}"
	fi
done

echo "build> Build finished.  Removing all installed packages left over from build..."
for pkgname in `${PKG_TOOLS_BIN}/pkg_info -e \*`
do
	if ${PKG_TOOLS_BIN}/pkg_info -qe "${pkgname}"; then
		pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"`
		case "${BULK_PREREQ}" in
			*"${pkgdir}"* )
				echo "Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
				;;
			* )
				echo ${PKG_TOOLS_BIN}/pkg_delete -r "${pkgname}"
				${PKG_TOOLS_BIN}/pkg_delete -r "${pkgname}"
				if ${PKG_TOOLS_BIN}/pkg_info -qe "${pkgname}"; then
					echo "$pkgname ($pkgdir) did not deinstall nicely.  Forcing the deinstall"
					${PKG_TOOLS_BIN}/pkg_delete -f "${pkgname}" || true
				fi
				;;
		esac
	fi
done

# for now, just quit if we were only mirroring distfiles.  At somepoint we
# should teach the post-build script to generate a nice report about how many
# distfiles were downloaded, how many had bad checksums, failed master sites,
# network speed, etc.

if [ "x$mirror_only" = "xyes" ]; then
	echo "Bulk mirror of distfiles completed: `date`"
	exit 0
fi

echo "build> Post processing bulk build results..."

# Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by
# them for post-build to run.
echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..."
for pkgdir in $BULK_PREREQ lang/perl5; do
	echo "build> Installing prerequisite package $pkgdir"
	( cd "${pkgsrc_dir}/${pkgdir}" \
	  && ${BMAKE} bulk-install
	) || die "Failed to install prerequisite packages."
done

#
# Generate the post-build report.
#
echo "build> Generating the bulk build report..."
BUILDDATE=`date +%Y-%m-%d`
mkdir -p "${FTP}"
( cd "${pkgsrc_dir}" \
  && ${PERL5} mk/bulk/post-build \
     > ${FTP}/pkgsrc-results-${BUILDDATE}.txt
) || die "Could not write the results file."

#
# Notify the ADMIN of the finished build.
#
case $noemail in
no)	cat "${FTP}/pkgsrc-results-${BUILDDATE}.txt" \
	| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results $BUILDDATE" "$ADMIN"
esac

# Done!
echo ""
echo "build> Bulk build ended: `date`"