pkgsrc/mk/bulk/bsd.bulk-pkg.mk
agc 1e90841f0d Add a new definition, _PRESERVE_WRKDIR, which defaults to "no". This
definition is not intended to be set by the user, and so has a leading
'_', and is undocumented.

If set to "yes", then a "make clean" of the package will not be done,
thereby preserving the working directory.  This is useful for
debugging problems in bulk builds.
2004-01-26 21:24:41 +00:00

391 lines
16 KiB
Makefile

# $NetBSD: bsd.bulk-pkg.mk,v 1.56 2004/01/26 21:24:41 agc Exp $
#
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@netbsd.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 3. All advertising materials mentioning features or use of this software
# must display the following acknowledgement:
# This product includes software developed by Hubert Feyrer for
# the NetBSD Project.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
##
## For this to work, put
## DEPENDS_TARGET= bulk-install
## BATCH= yes
## into your /etc/mk.conf, then cd /usr/pkgsrc; make bulk-package
##
## bulk-install will only compile pkgs if necessary, and install binary
## pkgs if possible/available.
## bulk-package will build binary packages, and remove them (together with
## any depending packages) immediately after that, leaving only the
## binary package back. Broken packages are marked (``.broken'').
##
LS?= ls
WC?= wc
# This variable is set to 'yes' by the pkgsrc/mk/bulk/build script. It enables
# the use of several cache files (DEPENDSTREEFILE, DEPENDSFILE, SUPPORTSFILE,
# and INDEXFILE) for speeding up the processing of dependencies. If we're not
# actively in the middle of a bulk build, however, we don't know that these files
# exist or are up to date and they take quite a while to rebuild. So unless
# they're known to exist and be up to date, don't use them.
USE_BULK_CACHE?= no
# Shall we remove any packages which are installed, but not required
# to build this package
PRECLEAN?= yes
# If OBJHOSTNAME is set, use first component of hostname in cache and log files
# If OBJMACHINE is set, use ${MACHINE_ARCH} in the cache and log files
.if defined(OBJHOSTNAME)
. if !defined(_HOSTNAME)
_HOSTNAME!= ${UNAME} -n
. endif
BULK_ID?= .${_HOSTNAME:C|\..*||}
.elif defined(OBJMACHINE)
BULK_ID?= .${MACHINE_ARCH}
.else
BULK_ID?=
.endif
# This file exists to mark a package as broken
BROKENFILE?= .broken${BULK_ID}.html
# This file is where the log of the build goes
BUILDLOG?= .make${BULK_ID}
# This is a top level file which lists the entire pkgsrc depends tree in the format:
# foo/bar devel/libfoo
# meaning 'foo/bar' is requied to build 'devel/libfoo'
# this is in the format needed by tsort(1)
DEPENDSTREEFILE?= ${_PKGSRCDIR}/.dependstree${BULK_ID}
# This is a top level file which lists the entire pkgsrc depends tree in the format:
# foo/bar depends on: devel/libfoo devel/libbar devel/baz .....
# ie, to build foo/bar we need devel/libfoo devel/libbar devel/baz ... installed
DEPENDSFILE?= ${_PKGSRCDIR}/.depends${BULK_ID}
# This is a top level file which lists the entire pkgsrc depends tree in the format:
# devel/libfoo is depended upon by: foo/bar graphics/gtkfoo ...
# ie, to build foo/bar we need devel/libfoo to be installed.
# to build graphics/gtkfoo we need devel/libfoo to be installed
SUPPORTSFILE?= ${_PKGSRCDIR}/.supports${BULK_ID}
# This is a top level file which cross-references each package name and pkg directory
# in the format:
# devel/libfoo libfoo-1.3
INDEXFILE?= ${_PKGSRCDIR}/.index${BULK_ID}
# file containing a list of all the packages in the correct order for a bulk build.
# the correct order is one where packages that are required by others are built
# before the packages which require them.
ORDERFILE?= ${_PKGSRCDIR}/.order${BULK_ID}
# file which is used as a timestamp for when the build started. This is used eventually
# for looking for leftover files (files not properly deinstalled)
STARTFILE?= ${_PKGSRCDIR}/.start${BULK_ID}
# a list of pkgs which we should _never_ delete during a build. The primary use is for digest
# and also for xpkgwedge. Add pkgtools/xpkgwedge in /etc/mk.conf to do an xpkgwedged bulk build.
BULK_PREREQ+= pkgtools/digest
# by default, clean up any broken packages
_PRESERVE_WRKDIR?= no
# build the cache files used as part of a full bulk build
# Note: we have to install the BULK_PREREQ packages _before_
# extracting the depends tree because some packages like
# xpkgwedge only become DEPENDS if it is installed
bulk-cache:
@${ECHO_MSG} "BULK> Installing BULK_PREREQ packages"
.for __prereq in ${BULK_PREREQ}
cd ${_PKGSRCDIR}/${__prereq} && ${MAKE} bulk-install
.endfor
@${ECHO_MSG} "BULK> Building complete pkgsrc dependency tree (this may take a while)."
cd ${_PKGSRCDIR} && ${SH} mk/bulk/printdepends ${BROKENFILE} > ${DEPENDSTREEFILE}
@${ECHO_MSG} "BULK> Sorting build order."
${TSORT} ${DEPENDSTREEFILE} > ${ORDERFILE}
@${ECHO_MSG} "BULK> Generating up and down dependency files."
${AWK} -f ${_PKGSRCDIR}/mk/bulk/tflat up ${DEPENDSTREEFILE} > ${SUPPORTSFILE}
${AWK} -f ${_PKGSRCDIR}/mk/bulk/tflat down ${DEPENDSTREEFILE} > ${DEPENDSFILE}
@${ECHO_MSG} "BULK> Generating package name <=> package directory cross reference file"
@${ECHO_MSG} " (this may take a while)."
cd ${_PKGSRCDIR} && ${SH} mk/bulk/printindex ${BROKENFILE} > ${INDEXFILE}
# remove the bulk cache files
clean-bulk-cache:
${RM} -f ${DEPENDSTREEFILE} \
${DEPENDSFILE} \
${SUPPORTSFILE} \
${INDEXFILE} \
${ORDERFILE}
# check if the $REF file is uptodate, i.e. is newer than any of
# the pkg files; prints "1" if upto date, "0" if not.
bulk-check-uptodate:
@uptodate=1 ; \
if [ -f "${REF}" ]; then \
${SHCOMMENT} "Check files of this package" ; \
newfiles="`${FIND} . -type f -newer "${REF}" -print | ${EGREP} -v -e ./work -e COMMENT -e DESCR -e README.html -e CVS -e '^\./\.' || ${TRUE}`" ; \
nnewfiles="`${FIND} . -type f -newer "${REF}" -print | ${EGREP} -v -e ./work -e COMMENT -e DESCR -e README.html -e CVS -e '^\./\.' | ${WC} -l`" ; \
if [ "$$nnewfiles" -gt 0 ]; then \
${ECHO_MSG} >&2 "BULK> Package ${PKGNAME} ($$newfiles) modified since last 'make package' re-packaging..." ; \
uptodate=0 ; \
else \
${ECHO_MSG} >&2 "BULK> ${REF} is up to date." ; \
fi ; \
else \
${ECHO_MSG} >&2 "BULK> Package ${PKGNAME} not built yet, packaging..." ; \
uptodate=0 ; \
fi ; \
if [ "$$uptodate" = "1" ]; then \
${SHCOMMENT} "Check required binary packages" ; \
deps=${DEPENDS:C/:.*//:Q} ; \
for dep in $$deps ; do \
${SHCOMMENT} "check against the binary pkg that pkg_add would pick, too:" ; \
${SHCOMMENT} "(Only one should be returned here, really...)" ; \
pkg=`${PKG_ADMIN} lsbest "${PACKAGES}/All/$$dep"` ; \
if [ -z "$$pkg" ]; then \
${ECHO_MSG} >&2 "BULK> Required binary package $$dep does not exist, rebuilding... " ; \
uptodate=0 ; \
elif [ -n "$$(${FIND} $$pkg -prune -newer ${REF} -print)" ]; then \
${ECHO_MSG} >&2 "BULK> Required binary package $$dep (`basename $$pkg`) is newer, rebuilding... " ; \
uptodate=0 ; \
else \
${ECHO_MSG} >&2 "BULK> Required binary package $$dep (`basename $$pkg`) is usable. " ; \
fi ; \
done ; \
fi ; \
${ECHO_MSG} $$uptodate
# rebuild binpkg if any of the pkg files is newer than the binary archive
# set DO to ":" to not actually do anything (debugging, ...)
bulk-package:
@if [ -f ${BROKENFILE} ]; then \
${ECHO_MSG} "BULK> *** Package ${PKGNAME} seems broken and needs attention:" ; \
${LS} -la ${BROKENFILE}; \
exit 1; \
fi
@( \
if [ "${PRECLEAN}" = "yes" ]; then \
${ECHO_MSG} '<pre>' ; \
${ECHO_MSG} '' ; \
${ECHO_MSG} '###' ; \
${ECHO_MSG} '###' `date`: ; \
${ECHO_MSG} '### ${MAKE} ${.TARGET} for ${PKGNAME}' ; \
${ECHO_MSG} '### Current pkg count: ' `${LS} -l ${PKG_DBDIR} | ${GREP} '^d' | ${WC} -l` installed packages: `${LS} ${PKG_DBDIR} | ${GREP} -v pkgdb.byfile.db`; \
${ECHO_MSG} '###' ; \
fi \
) 2>&1 | ${TEE} -a ${BUILDLOG}
@uptodate=`${MAKE} ${MAKEFLAGS} bulk-check-uptodate REF=${PKGFILE}` ; \
if ${PKG_INFO} -qe "${PKGNAME:C/-[^-]*$/-[0-9]*/}" ; then \
installed=1; \
else \
installed=0; \
fi ; \
if [ $$uptodate = 1 ]; then \
( if [ $$installed = 1 ]; then \
${ECHO_MSG} "BULK> Package ${PKGNAME} is upto-date, and still installed" ; \
else \
${ECHO_MSG} "BULK> Nothing to be done." ; \
fi \
) 2>&1 | ${TEE} -a ${BUILDLOG}; \
else \
( if [ $$installed = 1 ]; then \
${ECHO_MSG} "BULK> Removing outdated (installed) package ${PKGNAME} first." ; \
${ECHO_MSG} ${MAKE} deinstall ; \
${DO} ${MAKE} deinstall ; \
if ${PKG_INFO} -qe ${PKGWILDCARD} ; then \
${ECHO_MSG} ${PKG_DELETE} -r ${PKGWILDCARD} ;\
${DO} ${PKG_DELETE} -r ${PKGWILDCARD} ;\
fi ;\
fi ; \
if [ -f ${PKGFILE} ]; then \
${ECHO_MSG} "BULK> Removing old binary package..." ; \
${ECHO_MSG} ${RM} -f ${PKGFILE} ; \
${DO} ${RM} -f ${PKGFILE} ; \
for cat in ${CATEGORIES} ;\
do \
${ECHO_MSG} ${RM} -f ${PACKAGES}/$$cat/${PKGNAME}${PKG_SUFX}; \
${DO} ${RM} -f ${PACKAGES}/$$cat/${PKGNAME}${PKG_SUFX}; \
done ;\
fi; \
${ECHO_MSG} "BULK> Full rebuild in progress..." ; \
${ECHO_MSG} "BULK> Cleaning package and its depends" ;\
if [ "${USE_BULK_CACHE}" = "yes" ]; then \
for pkgdir in ${PKGPATH} `${GREP} "^${PKGPATH} " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \
${DO} (cd ${_PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \
done ;\
else \
${ECHO_MSG} ${MAKE} clean CLEANDEPENDS=YES;\
${DO} ${MAKE} clean CLEANDEPENDS=YES;\
fi; \
if [ "${PRECLEAN}" = "yes" ]; then \
${ECHO_MSG} "BULK> Removing installed packages which are not needed to build ${PKGNAME}" ; \
for pkgname in `${PKG_INFO} -e \*` ; \
do \
if [ "${USE_BULK_CACHE}" = "yes" ]; then \
pkgdir=`${GREP} " $$pkgname " ${INDEXFILE} | ${AWK} '{print $$1}'` ;\
if [ -z "$$pkgdir" ]; then \
pkgdir=unknown ; \
fi; \
if ${PKG_INFO} -qe $$pkgname ; then \
${SHCOMMENT} "Remove only unneeded pkgs" ; \
if ${EGREP} -q "^${PKGPATH} .* $$pkgdir( |$$)" ${DEPENDSFILE} ; then \
${ECHO_MSG} "BULK> ${PKGNAME} requires installed package $$pkgname ($$pkgdir) to build." ;\
else \
case "${BULK_PREREQ}" in \
*$$pkgdir* ) \
${ECHO_MSG} "BULK> Keeping BULK_PREREQ: $$pkgname ($$pkgdir)" ;\
;; \
* ) \
${ECHO_MSG} ${PKG_DELETE} -r $$pkgname ; \
${DO} ${PKG_DELETE} -r $$pkgname || true ; \
if ${PKG_INFO} -qe $$pkgname ; then \
${DO} ${PKG_DELETE} -f $$pkgname || true ; \
fi ;\
;; \
esac ; \
fi ;\
fi ;\
else \
${SHCOMMENT} "Remove all pkgs" ; \
${ECHO_MSG} ${PKG_DELETE} -r $$pkgname ; \
${DO} ${PKG_DELETE} -r $$pkgname || true ; \
if ${PKG_INFO} -qe $$pkgname ; then \
${DO} ${PKG_DELETE} -f $$pkgname || true ; \
fi ;\
fi ;\
done ; \
fi ;\
if [ "${USE_BULK_CACHE}" = "yes" ]; then \
${SHCOMMENT} "Install required depends via binarypkgs XXX" ; \
${ECHO_MSG} "BULK> Installing packages which are required to build ${PKGNAME}." ;\
for pkgdir in `${GREP} "^${PKGPATH} " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'` ${BULK_PREREQ} ; do \
pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ; \
if [ -z "$$pkgname" ]; then continue ; fi ;\
pkgfile=${PACKAGES}/All/$${pkgname}.tgz ;\
if ${PKG_INFO} -qe $$pkgname ; then \
${ECHO_MSG} "BULK> Required package $$pkgname ($$pkgdir) is already installed" ; \
else \
if [ -f $$pkgfile ]; then \
${ECHO_MSG} "BULK> ${PKG_ADD} ${PKG_ARGS_ADD} $$pkgfile"; \
${DO} ${PKG_ADD} ${PKG_ARGS_ADD} $$pkgfile || ${ECHO_MSG} "warning: could not add $$pkgfile." ; \
else \
${ECHO_MSG} "BULK> warning: $$pkgfile does not exist. It will be rebuilt." ;\
fi ;\
fi ;\
done ;\
fi ;\
if [ -f ${INTERACTIVE_COOKIE} ]; then \
${ECHO_MSG} "BULK> Removing old marker for INTERACTIVE_STAGE..." ; \
${ECHO_MSG} ${RM} -f ${INTERACTIVE_COOKIE} ; \
${DO} ${RM} -f ${INTERACTIVE_COOKIE} ; \
fi ;\
${ECHO_MSG} ${MAKE} package '(${PKGNAME})' 2>&1 ; \
${DO} ( ${MAKE} package 2>&1 ); \
) 2>&1 | ${TEE} -a ${BUILDLOG} ; \
if [ -f ${PKGFILE} ]; then \
${RM} ${BUILDLOG} ; \
else \
${MV} ${BUILDLOG} ${BROKENFILE} ;\
( \
${ECHO_MSG} "BULK> ${PKGNAME} was marked as broken:" ; \
${LS} -la ${BROKENFILE} ; \
${ECHO_MSG} ${MAKE} deinstall ; \
${DO} ${MAKE} deinstall ; \
nbrokenby=0;\
if [ "${USE_BULK_CACHE}" = "yes" ]; then \
${ECHO_MSG} "BULK> Marking all packages which depend upon ${PKGNAME} as broken:"; \
for pkgdir in `${GREP} "^${PKGPATH} " ${SUPPORTSFILE} | ${SED} -e 's;^.*:;;g'`; do \
pkgname=`${GREP} "^$$pkgdir " ${INDEXFILE} | ${AWK} '{print $$2}'` ;\
if [ -z "$$pkgname" ]; then pkgname=unknown ; fi ; \
${ECHO_MSG} "BULK> marking package that requires ${PKGNAME} as broken: $$pkgname ($$pkgdir)";\
pkgerr="-1"; \
pkgignore=`(cd ${_PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=PKG_FAIL_REASON)`; \
pkgskip=`(cd ${_PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=PKG_SKIP_REASON)`; \
if [ ! -z "$${pkgignore}$${pkgskip}" -a ! -f ${_PKGSRCDIR}/$$pkgdir/${BROKENFILE} ]; then \
${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) may not be packaged because:" >> ${_PKGSRCDIR}/$$pkgdir/${BROKENFILE};\
${ECHO_MSG} "BULK> $$pkgignore" >> ${_PKGSRCDIR}/$$pkgdir/${BROKENFILE};\
${ECHO_MSG} "BULK> $$pkgskip" >> ${_PKGSRCDIR}/$$pkgdir/${BROKENFILE};\
if [ -z "`(cd ${_PKGSRCDIR}/$$pkgdir && ${MAKE} show-var VARNAME=BROKEN)`" ]; then \
pkgerr="0"; \
else \
pkgerr="1"; \
fi; \
fi; \
${ECHO_MSG} "BULK> $$pkgname ($$pkgdir) is broken because it depends upon ${PKGNAME} (${PKGPATH}) which is broken." \
>> ${_PKGSRCDIR}/$$pkgdir/${BROKENFILE};\
${ECHO_MSG} "Please view the <a href=\"../../${PKGPATH}/${BROKENFILE}\">build log for ${PKGNAME}</a>" \
>> ${_PKGSRCDIR}/$$pkgdir/${BROKENFILE};\
nbrokenby=`expr $$nbrokenby + 1`;\
if ${GREP} -q " $$pkgdir/${BROKENFILE}" ${_PKGSRCDIR}/${BROKENFILE} ; then :; else \
${ECHO} " $$pkgerr $$pkgdir/${BROKENFILE} 0 " >> ${_PKGSRCDIR}/${BROKENFILE} ;\
fi ;\
done ;\
fi ;\
nerrors=`${GREP} -c '^\*\*\* Error code' ${BROKENFILE} || true`; \
if [ -f ${INTERACTIVE_COOKIE} ]; then \
nerrors="0"; \
fi; \
${ECHO_MSG} " $$nerrors ${PKGPATH}/${BROKENFILE} $$nbrokenby " >> ${_PKGSRCDIR}/${BROKENFILE} \
) 2>&1 | ${TEE} -a ${BROKENFILE}; \
fi ; \
case ${_PRESERVE_WRKDIR} in \
yes|YES) ;; \
*) ${ECHO_MSG} "BULK> Cleaning packages and its depends"; \
if [ "${USE_BULK_CACHE}" = "yes" ]; then \
for pkgdir in ${PKGPATH} `${GREP} "^${PKGPATH} " ${DEPENDSFILE} | ${SED} -e 's;^.*:;;g'`; do \
${DO} (cd ${_PKGSRCDIR}/$$pkgdir && ${MAKE} clean) ; \
done; \
else \
${ECHO_MSG} ${MAKE} clean CLEANDEPENDS=YES;\
${DO} ${MAKE} clean CLEANDEPENDS=YES; \
fi ;; \
esac; \
fi
@if [ ! -f ${PKGFILE} ]; then \
${ECHO_MSG} "BULK> Build for ${PKGNAME} was not successful, aborting." | ${TEE} -a ${BROKENFILE} ; \
false; \
else \
${RM} -f ${BUILDLOG} ;\
fi
# Install pkg - if possible from binary pkg (i.e. available & upto date)
# else revert to the old recompiling.
# Don't rely upon pkg_add to snarf in depending pkgs as they may have
# been modified and need rebuilding.
bulk-install:
@if [ `${MAKE} bulk-check-uptodate REF=${PKGFILE}` = 1 ]; then \
if ${PKG_INFO} -qe ${PKGNAME} ; then :; else \
${DO} ${MAKE} install-depends ; \
${ECHO_MSG} "BULK>" ${PKG_ADD} ${PKG_ARGS_ADD} ${PKGFILE} ; \
${DO} ${PKG_ADD} ${PKG_ARGS_ADD} ${PKGFILE} ; \
fi ; \
else \
${ECHO_MSG} ${MAKE} bulk-package PRECLEAN=no; \
${DO} ${MAKE} bulk-package PRECLEAN=no; \
fi