freebsd-ports/Makefile
Kris Kennaway 54e565eedc Major optimizations for 'make index' and other recursive traversal
targets.

* Use /rescue/sh for index builds instead of /bin/sh, when it exists.
  The former is statically linked and faster to execute, which becomes
  significant when executing it tens of thousands of times.  This
  trick can be used with other recursive targets by passing in
  __MAKE_SHELL.

* Get rid of make variable assignments that use != command invocations
  in the critical path, using several methods:

  - rewriting logic to use shell or make builtins instead of external command executions
  - macroizing commands and executing them in the targets where they
    are needed instead of with every invocation of make
  - precomputing the results of invariant commands in
    bsd.port.subdir.mk and passing them in explicitly to child makes,
    and using this to avoid recalculation in all the children. NB: the
    commands are still run one per top-level subdirectory but this
    does not currently seem to be a major issue.  They could be moved
    further up into the top-level Makefile at the cost of some
    cleanliness.
  - Committers are strongly discouraged from adding further "bare" !=
    assignments to the ports tree, even in their own ports.  One of
    the above strategies should be used to avoid future bloat.

* Rewrite the core 'describe' target to work entirely within a single
  shell process using only builtin commands.  The old version is
  retained as a backup for use on systems older than 603104, which
  does not have the make :u modifier.  This cuts down the number of
  processes executed during the course of a 'make index' by an order
  of magnitude, and we are essentially now amortized to the minimum of
  a single make + sh instance per port, plus whatever commands the
  port makefile itself executes (which are usually unnecessary and
  bogus).

* Less validation of the WWW: target is performed; this can become
  policed at a port level by portlint.  Specifically we look at the
  second word of the first line beginning with "WWW:" in pkg-descr,
  and append "http://" to it unless it already begins with "http://",
  "https://" or "ftp://".  Thanks to dougb for the idea of how to
  extract WWW: using shell builtins.

* Use the "true" shell builtin instead of echo > /dev/null for a
  measurable decrease in CPU use.

* Add a note about dubious escaping strategy in bsd.port.subdir.mk

* Minor change in output of 'make describe': it no longer strips
  trailing CR characters from pkg-descr files with MSDOS CR/LF
  termination.  Instead the makeindex perl script that post-processes
  make describe into the INDEX is tweaked to strip on input.

The bottom line is that on my test hardware INDEX builds are now
faster by more than a factor of 2 and with a reduction in system time
by a factor of 4-8 depending on configuration.
2008-07-19 17:59:41 +00:00

190 lines
5.9 KiB
Makefile
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# $FreeBSD$
#
SUBDIR += accessibility
SUBDIR += arabic
SUBDIR += archivers
SUBDIR += astro
SUBDIR += audio
SUBDIR += benchmarks
SUBDIR += biology
SUBDIR += cad
SUBDIR += chinese
SUBDIR += comms
SUBDIR += converters
SUBDIR += databases
SUBDIR += deskutils
SUBDIR += devel
SUBDIR += dns
SUBDIR += editors
SUBDIR += emulators
SUBDIR += finance
SUBDIR += french
SUBDIR += ftp
SUBDIR += games
SUBDIR += german
SUBDIR += graphics
SUBDIR += hebrew
SUBDIR += hungarian
SUBDIR += irc
SUBDIR += japanese
SUBDIR += java
SUBDIR += korean
SUBDIR += lang
SUBDIR += mail
SUBDIR += math
SUBDIR += mbone
SUBDIR += misc
SUBDIR += multimedia
SUBDIR += net
SUBDIR += net-im
SUBDIR += net-mgmt
SUBDIR += net-p2p
SUBDIR += news
SUBDIR += palm
SUBDIR += polish
SUBDIR += ports-mgmt
SUBDIR += portuguese
SUBDIR += print
SUBDIR += russian
SUBDIR += science
SUBDIR += security
SUBDIR += shells
SUBDIR += sysutils
SUBDIR += textproc
SUBDIR += ukrainian
SUBDIR += vietnamese
SUBDIR += www
SUBDIR += x11
SUBDIR += x11-clocks
SUBDIR += x11-drivers
SUBDIR += x11-fm
SUBDIR += x11-fonts
SUBDIR += x11-servers
SUBDIR += x11-themes
SUBDIR += x11-toolkits
SUBDIR += x11-wm
PORTSTOP= yes
.include <bsd.port.subdir.mk>
index:
@rm -f ${INDEXDIR}/${INDEXFILE}
@cd ${.CURDIR} && make ${INDEXDIR}/${INDEXFILE}
fetchindex: ${INDEXDIR}/${INDEXFILE}.bz2
@bunzip2 < ${INDEXDIR}/${INDEXFILE}.bz2 > ${INDEXDIR}/${INDEXFILE} && \
chmod a+r ${INDEXDIR}/${INDEXFILE}
${INDEXDIR}/${INDEXFILE}.bz2: .PHONY
@${FETCHINDEX} ${INDEXDIR}/${INDEXFILE}.bz2 ${MASTER_SITE_INDEX}${INDEXFILE}.bz2
MASTER_SITE_INDEX?= http://www.FreeBSD.org/ports/
SETENV?= /usr/bin/env
FETCHINDEX?= ${SETENV} ${FETCH_ENV} fetch -am -o
INDEX_JOBS?= 2
.if !defined(INDEX_VERBOSE)
INDEX_ECHO_MSG= true
INDEX_ECHO_1ST= echo -n
.else
INDEX_ECHO_MSG= echo 1>&2
INDEX_ECHO_1ST= echo
.endif
# /rescue/sh is statically linked and much faster to execute than the
# dynamically linked /bin/sh. This is significant for targets like
# make index that execute the shell tens of thousands of times.
.if exists(/rescue/sh)
INDEX_SHELL= /rescue/sh
.else
INDEX_SHELL= /bin/sh
.endif
${INDEXDIR}/${INDEXFILE}:
@${INDEX_ECHO_1ST} "Generating ${INDEXFILE} - please wait.."; \
if [ "${INDEX_PRISTINE}" != "" ]; then \
export LOCALBASE=/nonexistentlocal; \
export X11BASE=/nonexistentx; \
export USE_NONDEFAULT_X11BASE=1; \
fi; \
tmpdir=`/usr/bin/mktemp -d -t index` || exit 1; \
trap "rm -rf $${tmpdir}; exit 1" 1 2 3 5 10 13 15; \
( cd ${.CURDIR} && make -j${INDEX_JOBS} INDEX_TMPDIR=$${tmpdir} BUILDING_INDEX=1 \
__MAKE_SHELL=${INDEX_SHELL} \
ECHO_MSG="${INDEX_ECHO_MSG}" describe ) || \
(rm -rf $${tmpdir} ; \
if [ "${INDEX_QUIET}" = "" ]; then \
echo; \
echo "********************************************************************"; \
echo "Before reporting this error, verify that you are running a supported"; \
echo "version of FreeBSD (see http://www.FreeBSD.org/ports/) and that you"; \
echo "have a complete and up-to-date ports collection. (INDEX builds are"; \
echo "not supported with partial or out-of-date ports collections -- in"; \
echo "particular, if you are using cvsup, you must cvsup the \"ports-all\""; \
echo "collection, and have no \"refuse\" files.) If that is the case, then"; \
echo "report the failure to ports@FreeBSD.org together with relevant"; \
echo "details of your ports configuration (including FreeBSD version,"; \
echo "your architecture, your environment, and your /etc/make.conf"; \
echo "settings, especially compiler flags and WITH/WITHOUT settings)."; \
echo; \
echo "Note: the latest pre-generated version of INDEX may be fetched"; \
echo "automatically with \"make fetchindex\"."; \
echo "********************************************************************"; \
echo; \
fi; \
exit 1); \
cat $${tmpdir}/${INDEXFILE}.desc.* | (cd ${.CURDIR} ; perl ${.CURDIR}/Tools/make_index) | \
sed -e 's/ */ /g' -e 's/| */|/g' -e 's/ *|/|/g' -e 's./..g' | \
sort -t '|' +1 -2 | \
sed -e 's../.g' > ${INDEXDIR}/${INDEXFILE}.tmp; \
if [ "${INDEX_PRISTINE}" != "" ]; then \
sed -e "s,$${LOCALBASE},/usr/local," -e "s,$${X11BASE},/usr/X11R6," \
${INDEXDIR}/${INDEXFILE}.tmp > ${INDEXDIR}/${INDEXFILE}; \
else \
mv ${INDEXDIR}/${INDEXFILE}.tmp ${INDEXDIR}/${INDEXFILE}; \
fi; \
rm -rf $${tmpdir}; \
echo " Done."
print-index: ${INDEXDIR}/${INDEXFILE}
@awk -F\| '{ printf("Port:\t%s\nPath:\t%s\nInfo:\t%s\nMaint:\t%s\nIndex:\t%s\nB-deps:\t%s\nR-deps:\t%s\nE-deps:\t%s\nP-deps:\t%s\nF-deps:\t%s\nWWW:\t%s\n\n", $$1, $$2, $$4, $$6, $$7, $$8, $$9, $$11, $$12, $$13, $$10); }' < ${INDEXDIR}/${INDEXFILE}
CVS?= cvs
.if ${OSVERSION} >= 601101
SUP?= csup
.else
SUP?= cvsup
.endif
PORTSNAP?= portsnap
PORTSNAP_FLAGS?= -p ${.CURDIR}
.if defined(SUPHOST)
SUPFLAGS+= -h ${SUPHOST}
.endif
update:
.if defined(SUP_UPDATE) && defined(PORTSSUPFILE)
@echo "--------------------------------------------------------------"
@echo ">>> Running ${SUP}"
@echo "--------------------------------------------------------------"
@${SUP} ${SUPFLAGS} ${PORTSSUPFILE}
.elif defined(CVS_UPDATE)
@echo "--------------------------------------------------------------"
@echo ">>> Updating ${.CURDIR} from cvs repository" ${CVSROOT}
@echo "--------------------------------------------------------------"
cd ${.CURDIR}; ${CVS} -R -q update -A -P -d -I!
.else
@echo "--------------------------------------------------------------"
@echo ">>> Running ${PORTSNAP}"
@echo "--------------------------------------------------------------"
.if !exists(${PORTSDIR}/.portsnap.INDEX)
@echo "Error: 'make update' uses portsnap(8) by default and"
@echo "needs ${PORTSDIR} to be created by portsnap on its first run."
@echo "Please run 'portsnap fetch extract' first."
@echo "You can also define SUP_UPDATE and PORTSSUPFILE to use csup(1)"
@echo "or CVS_UPDATE to use cvs(1) for updating."
.else
@${PORTSNAP} ${PORTSNAP_FLAGS} fetch
@${PORTSNAP} ${PORTSNAP_FLAGS} update
.endif
.endif