pkgsrc/mk/bulk/build

#!/bin/sh
# $NetBSD: build,v 1.102 2007/06/15 15:52:22 rillig Exp $

#
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
#    must display the following acknowledgement:
#      This product includes software developed by Hubert Feyrer for
#	the NetBSD Project.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

##
## Globals
##
scriptdir=`dirname "$0"`
scriptdir=`cd "${scriptdir}" && pwd`

#
# Default values for command line options.
#
resume=no
mirror_only=no
target=bulk-package
makeargs=""
noemail=no
post_only=no
prepare_only=no

##
## Functions
##

usage () {
	cat <<EOF
usage: $0 [options]
       $0 -h | --help

Runs a bulk pkgsrc build.

The following options are supported:

   -c | --config <file>
	Load the following configuration file instead of the default
	one.

   -e | --no-email
	Don't send email when the bulk build is finished.

   -h | --help
	Displays this message.

   -m | --mirror_only
	Downloads all distfiles needed for the build but does not run
	the build.  IMPORTANT:  Note that this will still run all the
	pre-build stuff which involves removing all of your installed
	packages.

	The only difference between this option and a regular bulk build
	is that the packages are not actually built.

   -p | --post-build
        Run the post-build processing and generate the report only.

   --prepare
        Only generate the package database that will be used when bulk
        building the packages.

   -r | --resume
	Resume a previously interrupted bulk build.

	The --resume option may be combined with the --mirror_only
	option.

   -s | --specific-pkgs
	Sets SPECIFIC_PKGS=1 when building packages.  This option is
	used for building a subset of pkgsrc.

EOF
}

# print out error message and exit 1
die () {
	echo "$0: error:" 1>&2
	for i in "$@"; do
		echo "    $i" 1>&2
	done
	exit 1
}

# This function can be overridden in the build.conf file to change the
# output format of the bulk build. It is used in a pipe, so if you want
# the original output, just define post_filter_cmd() { cat; }.
#
# For more sophisticated output, you may use all the variables that this
# example function uses.
post_filter_cmd() {
	${SED} "s;^;`date '+%Y/%m/%d %H:%M:%S'`  ${percent} ${pkgdir} @ ${MACHINE_ARCH}> ;g"
}

# perform post-processing of the bulk-build results
do_post_build () {
	echo "build> Post processing bulk build results..."

	# Re-install BULK_PREREQ as we may need functionality (e.g. SMTP) provided by
	# them for post-build to run.
	echo "build> Re-installing prerequisite packages specified with BULK_PREREQ..."
	for pkgdir in $BULK_PREREQ lang/perl5; do
		echo "build> Installing prerequisite package $pkgdir"
		( cd "${pkgsrc_dir}/${pkgdir}" \
		  && ${BMAKE} bulk-install
		) || die "Failed to install prerequisite packages."
	done

	#
	# Generate the post-build report.
	#
	echo "build> Generating the bulk build report..."

	bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
	|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."

	report_dir="${REPORTS_DIR}/${bulk_build_id}"
	${MKDIR} "${report_dir}"
	( cd "${pkgsrc_dir}" \
  	  && ${PERL5} mk/bulk/post-build \
     	  > "${report_dir}/${REPORT_TXT_FILE}"
	) || die "Could not write the results file."
}

# Notify the ADMIN of the finished build.
do_email () {
	case $noemail in
	no)	cat "${report_dir}/${REPORT_TXT_FILE}" \
		| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results ${bulk_build_id}" "$ADMIN"
	esac
}

# output final note that we're done
do_done () {
	echo ""
	echo "build> Bulk build ended: `date`"
}

# set all commonly used variables, prepare files etc.
do_common_setup () {

	#
	# Choose an appropriate value for BMAKE depending on the operating
	# system.
	#
	opsys=`uname -s`
	case "$opsys" in
		NetBSD)	BMAKE=make ;;
		*)	BMAKE=bmake ;;
	esac
	export BMAKE

	#
	# Set resource limits as high as possible
	#
	ulimit -S -s `ulimit -H -s` # XXX: why do we need unlimited stack?
	ulimit -S -d `ulimit -H -d` # some processes grow rather large
	ulimit -S -t `ulimit -H -t` # pkgsrc bulk builds need _much_ time

	#
	# Find the configuration file.
	#
	BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
	case $BULK_BUILD_CONF in
		/*) ;;
		 *)	BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
	 esac

	#
	# Load the variables from the configuration file.
	#
	{ test -f "${BULK_BUILD_CONF}"		\
	  && . "${BULK_BUILD_CONF}"		\
	  && . "${scriptdir}/post-build-conf"	\
	  && check_config_vars 			\
	  && export_config_vars
	} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."

	#
	# Set the paths to commonly used directories.
	#
	pkgsrc_dir="${USR_PKGSRC}"
	pkglint_dir="${USR_PKGSRC}/pkgtools/pkglint"

	#
	# Set up variables specific for the bulk build.
	#
	BATCH="1"
	DEPENDS_TARGET="bulk-install"
	export BATCH DEPENDS_TARGET

	#
	# Unset some environment variables that could disturbe the build.
	#
	unset CDPATH || true	# ensure cd does not print new cwd to stdout, which
				# confuses the printindex script.
	unset DISPLAY || true	# allow sane failure for gimp, xlispstat
}

# Check that the package tools are up to date.
check_tools () {
	echo "build> Checking if the pkgtools are up-to-date"
	( cd "${pkglint_dir}"				\
  	  && ${BMAKE} fetch >/dev/null 2>&1
	) || {
	  echo "build> Updating pkgtools"
	  ( cd "${pkgsrc_dir}/pkgtools/pkg_install"	\
	    && ${BMAKE} clean 				\
	    && ${BMAKE} install 			\
	    && ${BMAKE} clean
	  ) || die "Could not update the package tools."
	}
}

# Run the pre-build script if necessary.
run_pre_build () {
	case $resume in
		yes)	echo "build> Resuming -- skipping pre-build script";;
		*)	# make veryveryclean :)
			( cd "${pkgsrc_dir}"		\
	  		  && /bin/sh mk/bulk/pre-build
			) || die "Error during bulk-build preparations, aborting.";;
	esac
}

# Load pkgsrc variables that affect the build process.
load_vars () {
	echo "+----------------------------------------+"
	echo "| Some variables used in the bulk build: |"
	echo "+----------------------------------------+"

	vars="	OPSYS OS_VERSION MACHINE_ARCH
		BULK_PREREQ
		BULKFILESDIR
		BULK_DBFILE DEPENDSFILE INDEXFILE ORDERFILE STARTFILE
		SUPPORTSFILE BULK_BUILD_ID_FILE BUILDLOG BROKENFILE
		BROKENWRKLOG
		AWK GREP MAIL_CMD MKDIR PERL5 SED
		PKG_DELETE PKG_INFO PKGBASE"

	values=`cd "$pkglint_dir" && $BMAKE show-vars VARNAMES="$vars" USE_TOOLS="awk grep mail mkdir perl sed"`

	for v in $vars; do
		eval "read $v" || die "Could not read value for $v"
		eval "value=\$$v"
		if [ "$v" != "BULK_PREREQ" ] && [ ! "$value" ]; then
			die "$v must not be empty."
		fi
		printf "%-15s = %s\\n" "$v" "$value"
	done <<EOF
$values
EOF
	echo "------------------------------------------"
	[ "$PKGBASE" = "pkglint" ] \
	|| die	"Error reading the variables." \
		"Try running $0 with another shell."

	# Get the location of commonly used files
	main_buildlog="${BULKFILESDIR}/${BUILDLOG}"
}

# Install prerequisite packages.
#
# Note: we do this _before_ the depends tree because some packages like
# xpkgwedge only become DEPENDS if they are installed.
install_prereqs () {
	echo "build> Installing prerequisite packages specified with BULK_PREREQ..."
	for pkgdir in $BULK_PREREQ; do
		echo "build> Installing prerequisite package $pkgdir"
		( cd "${pkgsrc_dir}/${pkgdir}"		\
	  	  && ${BMAKE} bulk-install
		) || die "Could not install prerequisite packages."
	done
}

# Everything is prepared. We can start building the real packages now.
#
# Loop over every package in the correct order.  Before building
# each one, check to see if we've already processed this package
# before.  This could happen if the build got interrupted and we
# started it again with the '-resume' option.  This prevents us
# from having to do a potentially very large number of make's to
# get back to where we let off.  After we build each package, add
# it to the top level buildlog
# (usually '.make' or '.make.${MACHINE}').  As a side benefit, this
# can make a progress-meter very simple to add!
do_real_bulk_build () {

	cd "${pkgsrc_dir}" || die "The pkgsrc directory does not exist."

	echo "build> Starting actual build using the order specified in $ORDERFILE..."

	# make sure we have something to grep in in the build loop
	touch "${main_buildlog}" || die "Cannot write to ${main_buildlog}."

	tot=`${AWK} 'END { print NR }' "${ORDERFILE}"`
	for pkgdir in `cat "${ORDERFILE}"`
	do
		if ${GREP} -q "^${pkgdir}\$" "${main_buildlog}"; then
			: "skip this package"
		else
			percent=`${AWK} -v tot="${tot}" '
				END {
					printf("%d/%d=%4.1f%%", NR, tot, NR*100/tot);
				}' "${main_buildlog}"`
			( cd "${pkgsrc_dir}/${pkgdir}" \
			  && ${NICE_LEVEL} ${BMAKE} USE_BULK_CACHE=yes "${target}" \
				$makeargs </dev/null | post_filter_cmd
			) || true
			echo "$pkgdir" >> "${main_buildlog}"
		fi
	done

	echo "build> Build finished."
}

# clean up installed packages left over
do_bulk_cleanup () {

	echo "build> Removing all installed packages left over from build..."
	for pkgname in `${PKG_INFO} -e \*`
	do
		if ${PKG_INFO} -qe "${pkgname}"; then
			pkgdir=`${AWK} '$2 == "'"$pkgname"'" { print $1; }' "$INDEXFILE"`
			case "${BULK_PREREQ}" in
				*"${pkgdir}"* )
					echo "build> Keeping BULK_PREREQ: $pkgname ($pkgdir)" ;
					;;
				* )
					echo "build> ${PKG_DELETE} -r ${pkgname}"
					${PKG_DELETE} -r "${pkgname}"
					if ${PKG_INFO} -qe "${pkgname}"; then
						echo "build> $pkgname ($pkgdir) did not deinstall nicely.  Forcing the deinstall"
						${PKG_DELETE} -f "${pkgname}" || true
					fi
					;;
			esac
		fi
	done
}

# start the full bulk-build
do_bulk_build () {
	echo "build> Bulk build started: `date`"
	echo ""

	# this function from post-build-conf
	show_config_vars

	check_tools
	run_pre_build
	load_vars

	#
	# Create the directory for the log files if necessary
	#
	if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
		${MKDIR} "${BULKFILESDIR}"
	fi

	#
	# Save the bulk build ID in a file, as it most often contains a time
	# stamp.
	#
	case $resume in
	no)	echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
		|| die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
	esac

	install_prereqs

	#
	# Create the bulk cache files.
	#
	if [ "x$resume" != "xyes" ]; then
		( cd "${pkgsrc_dir}"				\
	  	  && env PKGLIST="${PKGLIST-}" ${BMAKE} bulk-cache $makeargs
		) || die "Could not create the bulk build cache."
	else
		if [ ! -f "${ORDERFILE}" ]; then
			die	"The ${ORDERFILE} does not exist." \
				"(You cannot resume a bulk build that has not yet started.)"
		fi
	fi

	# XXX: This looks like a hack, and indeed, the functions in this file
	# should be reorganized to better reflect the phases of the bulk build.
	if [ $prepare_only = yes ]; then
		exit 0
	fi

	do_real_bulk_build
	do_bulk_cleanup
}


##
## main
##

#
# Parse the command line.
#
while test $# -gt 0; do
	case $1 in
	-c|--config)
		shift
		BULK_BUILD_CONF=$1; shift
		;;
	-e|--no-email)
		noemail=yes
		shift
		;;
	-h|--help)
		usage
		exit 0
		;;
	-m|--mirror_only)
		mirror_only=yes
		target=mirror-distfiles
		shift
		;;
	-p|--post-build)
		post_only=yes
		shift
		;;
	--prepare)
		prepare_only=yes
		shift
		;;
	-r|--resume|--restart|restart)
		resume=yes
		shift
		;;
	-s|--specific-pkgs)
		makeargs="$makeargs SPECIFIC_PKGS=1"
		shift
		;;
	*)
		echo "unknown option:  $1" 1>&2
		usage 1>&2
		exit 1
		;;
	esac
done

do_common_setup

if [ "x$post_only" = "xyes" ]; then
	load_vars
	do_post_build
	exit 0
fi

do_bulk_build

# for now, just quit if we were only mirroring distfiles.  At somepoint we
# should teach the post-build script to generate a nice report about how many
# distfiles were downloaded, how many had bad checksums, failed master sites,
# network speed, etc.
if [ "x$mirror_only" = "xyes" ]; then
	echo "build> Bulk mirror of distfiles completed: `date`"
	exit 0
fi

do_post_build
do_email
do_done