freebsd-ports/Tools/portbuild/scripts/dopackages
2004-03-08 01:38:34 +00:00

625 lines
18 KiB
Bash
Executable file

#!/bin/sh
# configurable variables
pb=/var/portbuild
arch=$1
shift
. ${pb}/${arch}/portbuild.conf
. ${pb}/scripts/buildenv
# packages that take very long to build -- try to start building these first
quickports="lang/ghc games/civ2demo games/rt2-demo x11/XFree86-4 editors/openoffice-1.1 games/hlserver-cs misc/heyu x11/gnome2 x11/kde3"
status=${pb}/${arch}/status
errorexit () {
echo "$1" > ${status}
exit $1
}
usage () {
echo "usage: [-continue] [-nofinish] [-finish] [-restart] [-ftp] [-cdrom] [-nodummy] [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-noportscvs] [-norestr] [-plistcheck] [-distfiles] [-fetch-original] branch date"
errorexit 1
}
# usage: makeindex pb arch scripts branch user
makeindex () {
pb=$1
arch=$2
scripts=$3
branch=$4
user=$5
cd ${pb}/${arch}/${branch}/ports
echo "================================================"
echo "generating index"
echo "================================================"
echo "index generation started at $(date)"
${scripts}/makeindex ${arch} ${branch} || errorexit 1
echo "index generation ended at $(date)"
echo $(wc -l <${INDEXFILE}) "lines in INDEX"
chown ${user} ${INDEXFILE}
}
# usage: checkindex pb arch branch
checkindex () {
pb=$1
arch=$2
branch=$3
cd ${pb}/${arch}/${branch}/ports
if grep -q non-existent ${INDEXFILE}; then
echo "errors in INDEX:"
grep -n non-existent ${INDEXFILE}
errorexit 1
# grep -C non-existent ${INDEXFILE}
# grep -v non-existent ${INDEXFILE} > ${INDEXFILE}.tmp
# mv -f ${INDEXFILE}.tmp ${INDEXFILE}
fi
if ! awk -F '|' '{if (NF != 10) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' ${INDEXFILE}; then
echo "error in INDEX"
errorexit 1
fi
}
# usage: makeduds pb arch scripts branch
makeduds () {
pb=$1
arch=$2
scripts=$3
branch=$4
cd ${pb}/${arch}/${branch}/ports
echo "================================================"
echo "generating duds"
echo "================================================"
echo "duds generation started at $(date)"
if ! ${scripts}/makeduds ${arch} ${branch}; then
echo "error(s) detected, exiting script at $(date). Failed duds list was:"
cat ${pb}/${arch}/${branch}/duds
errorexit 1
fi
echo "duds generation ended at $(date)"
echo $(wc -l < ${pb}/${arch}/${branch}/duds) "items in duds"
echo "duds diff:"
diff ${pb}/${arch}/${branch}/duds.old ${pb}/${arch}/${branch}/duds
cp -p ${pb}/${arch}/${branch}/duds ${pb}/${arch}/${branch}/duds.old
cp -p ${pb}/${arch}/${branch}/duds ${pb}/${arch}/${branch}/duds.orig
}
# usage: setupnode pb arch scripts branch me node md5 tmpdir
setupnode () {
pb=$1
arch=$2
scripts=$3
branch=$4
me=$5
node=$6
md5=$7
tmpdir=$8
echo "setting up of $node started at $(date)"
scp -p ${scripts}/setupnode root@${node}:/tmp
ssh -n root@${node} /tmp/setupnode ${me} ${pb} ${arch} ${branch} ${tmpdir} ${md5}
echo "setting up of $node ended at $(date)"
}
# usage: restrictedlist pb scripts branch
restrictedlist () {
pb=$1
arch=$2
scripts=$3
branch=$4
cd ${pb}/${arch}/${branch}/ports
echo "================================================"
echo "creating restricted list"
echo "================================================"
echo "restricted list generation started at $(date)"
make ECHO_MSG=/usr/bin/true clean-restricted-list \
| sed -e "s./usr/ports/packages/.${pb}/${arch}/${branch}/packages/.g" \
-e "s./usr/ports/.${pb}/${arch}/${branch}/ports/.g" \
> ${pb}/${arch}/${branch}/restricted.sh
echo "restricted list generation ended at $(date)"
echo $(grep -c '^#' ${pb}/${arch}/${branch}/restricted.sh) "ports in ${pb}/${arch}/${branch}/restricted.sh"
}
# usage: cdromlist pb scripts branch
cdromlist () {
pb=$1
branch=$2
scripts=$3
branch=$4
cd ${pb}/${arch}/${branch}/ports
echo "================================================"
echo "creating cdrom list"
echo "================================================"
echo "cdrom list generation started at $(date)"
make ECHO_MSG=/usr/bin/true clean-for-cdrom-list \
| sed -e "s./usr/ports/distfiles/./distfiles/.g" \
-e "s./usr/ports/./${branch}/.g" \
> ${pb}/${arch}/${branch}/cdrom.sh
echo "cdrom list generation ended at $(date)"
echo $(grep -c '^#' ${pb}/${arch}/${branch}/cdrom.sh) "ports in ${pb}/${arch}/${branch}/cdrom.sh"
}
# usage: archiveports pb branch
archiveports () {
pb=$1
arch=$2
branch=$3
echo "started archive of ${pb}/${arch}/${branch}/ports at $(date)"
cd ${pb}/${arch}/${branch}
tar --exclude CVS -czf ${pb}/${arch}/${branch}/tarballs/ports.tar.gz ports
echo "ended archive of ${pb}/${arch}/${branch}/ports at $(date)"
}
# usage: generatemd5 pb branch
generatemd5 () {
pb=$1
arch=$2
branch=$3
echo "started generating CHECKSUM.MD5 at $(date)"
cd ${pb}/${arch}/${branch}/packages/All
find . -name '*.t[bg]z' | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5
echo "ended generating CHECKSUM.MD5 at $(date)"
}
scripts=${pb}/scripts
umask 002
me=$(hostname -s)
echo "Subject: $me package building logs"
echo
echo "Called with arguments: "${1+"$@"}
echo "Started at $(date)"
starttime=$(date +%s)
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/X11R6/bin
if [ $# = 0 ]; then
usage
fi
nobuild=0
noindex=0
noduds=0
nocvsup=0
nocvs=0
noportscvs=0
norestr=0
plistcheck=0
nodummy=0
cdrom=0
ftp=0
restart=0
cont=0
finish=0
nofinish=0
dodistfiles=0
fetch_orig=0
# optional arguments
while [ $# -gt 2 ]; do
case "x$1" in
x-nobuild)
nobuild=1
;;
x-noindex)
noindex=1
;;
x-noduds)
noduds=1
;;
x-cdrom)
cdrom=1
;;
x-nocvsup)
nocvsup=1
;;
x-nocvs)
nocvs=1
;;
x-noportscvs)
noportscvs=1
;;
x-norestr)
norestr=1
;;
x-plistcheck)
plistcheck=1
;;
x-nodummy)
nodummy=1
;;
x-ftp)
ftp=1
;;
x-distfiles)
dodistfiles=1
;;
x-fetch-original)
fetch_orig=1
;;
x-continue)
cont=1
;;
x-restart)
restart=1
;;
x-nofinish)
nofinish=1
;;
x-finish)
nobuild=1
finish=1
;;
*)
usage
;;
esac
shift
done
if [ "$restart" = 1 -o "$cont" = 1 -o "$finish" = 1 ]; then
skipstart=1
else
skipstart=0
fi
# mandatory arguments
branch=$1
date=$2
buildenv ${pb} ${arch} ${branch}
if [ "x$branch" != x4 -a "x$branch" != x4-exp -a "x$branch" != x5 ]; then
usage
fi
if [ "$norestr" = 1 ]; then
export NO_RESTRICTED=1
fi
if [ "$dodistfiles" = 1 ]; then
export WANT_DISTFILES=1
fi
if [ "$plistcheck" = 1 ]; then
export PLISTCHECK=1
fi
if [ "$nodummy" = 1 ]; then
export NODUMMY=1
fi
if [ "$cdrom" = 1 ]; then
export FOR_CDROM=1
fi
if [ "$fetch_orig" = 1 ]; then
export FETCH_ORIGINAL=1
fi
export DISTDIR=${pb}/${arch}/${branch}/distfiles
export PACKAGES=${pb}/${arch}/${branch}/packages
if [ "$skipstart" = 0 ]; then
if [ "$noportscvs" = 0 ]; then
echo "================================================"
echo "running cvs update -PAd on ${PORTSDIR}"
echo "================================================"
cd ${PORTSDIR}
cvs -qR update -PAd
# XXX Check for conflicts
fi
date > ${pb}/${arch}/${branch}/cvsdone
echo "================================================"
echo "running cvs update on ${pb}/${arch}/doc"
echo "================================================"
cd ${pb}/${arch}/${branch}/doc
cvs -qR update -PdA
# XXX Check for conflicts
if [ "$nocvs" = 0 ]; then
echo "================================================"
echo "running cvs update on ${SRCBASE}"
echo "================================================"
cd ${SRCBASE}
cvs -qR update -Pd
# XXX Check for conflicts
fi
echo "================================================"
echo "running make checksubdirs"
echo "================================================"
cd ${PORTSDIR}
make checksubdirs
# this one not in background to check return status
if [ "$noduds" = 0 ]; then
makeduds ${pb} ${arch} ${scripts} ${branch}
fi
if [ "$noindex" = 0 ]; then
makeindex ${pb} ${arch} ${scripts} ${branch} ${user} &
fi
fi
if [ "$nobuild" = 0 ]; then
md5=$(/sbin/md5 ${pb}/${arch}/${branch}/tarballs/bindist.tar | awk '{print $4}')
echo "================================================"
echo "setting up nodes"
echo "================================================"
for node in $(awk '{print $1}' ${pb}/${arch}/mlist); do
setupnode ${pb} ${arch} ${scripts} ${branch} ${me} ${node} ${md5} ${scratchdir} &
sleep 2
done
fi
if [ "$skipstart" = 0 ]; then
restrictedlist ${pb} ${arch} ${scripts} ${branch} &
sleep 2
if [ "$cdrom" = 1 ]; then
cdromlist ${pb} ${arch} ${scripts} ${branch} &
fi
fi
wait
if [ "$skipstart" = 0 ]; then
checkindex ${pb} ${arch} ${branch}
${pb}/scripts/makeparallel ${arch} ${branch}
if [ "$nodummy" = 0 ]; then
dummypkgs=""
for i in ${dummyports}; do
if [ -d $i ]; then
dummypkgs="${dummypkgs} $(cd $i; make package-name)${PKGSUFFIX}"
else
echo "dummy port directory \"$i\" does not exist -- skipping"
fi
done
fi
# hack to extend length of dependency chain to build quickports first
# XXX Does this actually work? - kk
for i in ${quickports}; do
if [ -d $i ]; then
quickpkg="$(cd $i; make package-name)${PKGSUFFIX}"
echo "all: dummy1-$quickpkg" >> ${pb}/${arch}/${branch}/Makefile
echo "dummy1-$quickpkg: dummy2-$quickpkg" >> ${pb}/${arch}/${branch}/Makefile
echo "dummy2-$quickpkg: dummy3-$quickpkg" >> ${pb}/${arch}/${branch}/Makefile
echo "dummy3-$quickpkg: $quickpkg" >> ${pb}/${arch}/${branch}/Makefile
else
echo "quick port directory \"$i\" does not exist -- skipping"
fi
done
cd ${pb}/${arch}
if [ "$nobuild" = 0 ]; then
#mkdir -p ${branch}/distfiles
#chown -R ${user} ${branch}/distfiles
cd ${pb}/${arch}/${branch}/bak
rm -rf errors logs packages old-errors
cd ${pb}/${arch}/${branch}
# need to preserve symlinks for packages but not for errorlogs
mv -f packages make.* tarballs/ports.tar.gz bak
cp -rp errors logs old-errors bak
rm -rf errors logs old-errors
shortdate=$(echo ${date} | sed -e 's/..$//')
cd ${pb}/${arch}/${branch}
mkdir -p packages/All
mkdir -p ${pb}/${arch}/archive/errorlogs
rm -rf ${pb}/${arch}/archive/errorlogs/e.${branch}.${date} ${pb}/${arch}/archive/errorlogs/e.${branch}.${shortdate}
mkdir -p ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}
ln -sf ${pb}/${arch}/archive/errorlogs/e.${branch}.${date} ${pb}/${arch}/${branch}/errors
ln -sf e.${branch}.${date} ${pb}/${arch}/archive/errorlogs/e.${branch}.${shortdate}
mkdir -p ${pb}/${arch}/${branch}/errors/old-errors
rm -rf ${pb}/${arch}/archive/errorlogs/a.${branch}.${date} ${pb}/${arch}/archive/errorlogs/a.${branch}.${shortdate}
mkdir -p ${pb}/${arch}/archive/errorlogs/a.${branch}.${date}
ln -sf ${pb}/${arch}/archive/errorlogs/a.${branch}.${date} ${pb}/${arch}/${branch}/logs
ln -sf a.${branch}.${date} ${pb}/${arch}/archive/errorlogs/a.${branch}.${shortdate}
# chown -RL ${user} errors logs packages
echo "error logs in ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}"
cp -p ${pb}/${arch}/${branch}/cvsdone ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}/cvsdone
cp -p ${pb}/${arch}/${branch}/cvsdone ${pb}/${arch}/archive/errorlogs/a.${branch}.${date}/cvsdone
cp -p ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/archive/errorlogs/e.${branch}.${date}/INDEX
cp -p ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/archive/errorlogs/a.${branch}.${date}/INDEX
if [ "$nodummy" = 0 ]; then
for dir in ${dummyports}; do
pkgname=$(cd ${pb}/${arch}/${branch}/ports/$dir; make package-name)${PKGSUFFIX}
if [ -f ${pb}/${arch}/${branch}/tarballs/${pkgname} ]; then
cp -p ${pb}/${arch}/${branch}/tarballs/${pkgname} packages/All
else
echo "Dummy package ${pkgname} does not exist!"
fi
done
fi
fi
fi
if [ "$nobuild" = 0 ]; then
cd ${pb}/${arch}/${branch}
if [ "$cont" = 1 ]; then
find errors/ -name \*.log | sed -e 's,\.log$,,' -e 's,^errors/,,' >> duds.errors
cat duds duds.errors | sort -u > duds.new
mv duds.new duds
else
cp duds.orig duds
fi
count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/${arch}/mlist | sed -e 's/\..*$//')
cd ${pb}/${arch}/${branch}/packages/All
ln -sf ../../Makefile .
echo "================================================"
echo "building packages (phase 1)"
echo "================================================"
echo "started at $(date)"
phase1start=$(date +%s)
make -k -j$count all > ../../make.0 2>&1 </dev/null
echo "ended at $(date)"
phase1end=$(date +%s)
echo "phase 1 took $(date -u -j -r $(($phase1end - $phase1start)) | awk '{print $4}')"
echo $(echo $(ls -1 ${pb}/${arch}/${branch}/packages/All | wc -l) - 2 | bc) "packages built"
echo $(echo $(du -sk ${pb}/${arch}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
echo $(echo $(du -sk ${pb}/${arch}/${branch}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
cd ${pb}/${arch}/${branch}
if grep -qE '(ptimeout|pnohang): killing' make.0; then
echo "The following port(s) timed out:"
grep -E '(ptimeout|pnohang): killing' make.0 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/'
fi
ls -asFlrt ${pb}/${arch}/${branch}/packages/All > ${pb}/${arch}/${branch}/logs/ls-lrt-1
cp -rp errors old-errors
cd ${pb}/${arch}/${branch}/old-errors
${pb}/scripts/processlogs
md5=$(/sbin/md5 ${pb}/${arch}/${branch}/tarballs/bindist.tar | awk '{print $4}')
echo "================================================"
echo "setting up nodes"
echo "================================================"
for node in $(awk '{print $1}' ${pb}/${arch}/mlist); do
setupnode ${pb} ${arch} ${scripts} ${branch} ${me} ${node} ${md5} ${scratchdir} &
sleep 2
done
wait
echo "setting up of nodes ended at $(date)"
count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/${arch}/mlist | sed -e 's/\..*$//')
cd ${pb}/${arch}/${branch}/packages/All
echo "================================================"
echo "building packages (phase 2)"
echo "================================================"
echo "started at $(date)"
phase2start=$(date +%s)
make -k -j$count all > ../../make.1 2>&1 </dev/null
echo "ended at $(date)"
phase2end=$(date +%s)
echo "phase 2 took $(date -u -j -r $(($phase2end - $phase2start)) | awk '{print $4}')"
echo $(ls -1 ${pb}/${arch}/${branch}/packages/All | wc -l) "packages built"
echo $(cat ${pb}/${arch}/${branch}/packages/${INDEXFILE} | wc -l) "lines in INDEX"
echo $(echo $(du -sk ${pb}/${arch}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
echo $(echo $(du -sk ${pb}/${arch}/${branch}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
fi
cd ${pb}/${arch}/${branch}
if grep -qE '(ptimeout|pnohang): killing' make.1; then
echo "The following port(s) timed out:"
grep -E '(ptimeout|pnohang): killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/'
fi
# Clean up temporary duds file
if [ "$cont" = 1 ]; then
cp duds.orig duds
fi
cd ${pb}/${arch}/${branch}/packages/All
if [ "$nofinish" = 0 ]; then
rm -f Makefile
# Always delete restricted packages/distfiles since they're published on the
# website
echo "deleting restricted ports"
sh ${pb}/${arch}/${branch}/restricted.sh
if [ "$cdrom" = 1 ]; then
echo "deleting cdrom restricted ports"
sh ${pb}/${arch}/${branch}/cdrom.sh
fi
# Remove packages not listed in INDEX
${pb}/scripts/prunepkgs ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages
fi
#rm -rf ${pb}/${arch}/${branch}/bad
#mkdir -p ${pb}/${arch}/${branch}/bad
#echo "checking packages"
#for i in *${PKGSUFFIX}; do
# if ! ${PKGZIPCMD} -t $i; then
# echo "Warning: package $i is bad, moving to ${pb}/${arch}/${branch}/bad"
# # the latest link will be left behind...
# mv $i ${pb}/${arch}/${branch}/bad
# rm ../*/$i
# fi
#done
if [ "$nofinish" = 0 ]; then
# Remove INDEX entries for packages that do not exist
${pb}/scripts/chopindex ${pb}/${arch}/${branch}/ports/${INDEXFILE} ${pb}/${arch}/${branch}/packages > ${pb}/${arch}/${branch}/packages/INDEX
cd ${pb}/${arch}/${branch}/old-errors
for i in *.log; do
cp -p ${i} ${pb}/${arch}/${branch}/errors/old-errors;
done
archiveports ${pb} ${arch} ${branch} &
generatemd5 ${pb} ${arch} ${branch} &
wait
ls -asFlrt ${pb}/${arch}/${branch}/packages/All > ${pb}/${arch}/${branch}/logs/ls-lrt
cp -p ${pb}/${arch}/${branch}/make.[01] ${pb}/${arch}/${branch}/logs
# Always delete restricted distfiles
echo "deleting restricted distfiles"
sh ${pb}/${arch}/${branch}/restricted.sh
if [ "$cdrom" = 1 ]; then
echo "deleting cdrom restricted distfiles"
sh ${pb}/${arch}/${branch}/cdrom.sh
fi
echo "================================================"
echo "copying distfiles"
echo "================================================"
echo "started at $(date)"
cd ${pb}/${arch}
#rm -rf ${branch}/bak/distfiles
#mv -f ${branch}/distfiles ${branch}/bak
${scripts}/dodistfiles ${pb}/${arch}/${branch}/distfiles
rm -rf ${pb}/${arch}/${branch}/distfiles/.pbtmp
rm -f ${pb}/${arch}/${branch}/distfiles/.done
if [ "$branch" != "4-exp" ]; then
# Currently broken - kk
#su ${user} -c "${scripts}/cpdistfiles ${branch} > ${pb}/${arch}/${branch}/cpdistfiles.log 2>&1 </dev/null" &
if [ "$ftp" = 1 ]; then
echo "ended at $(date)"
echo "================================================"
echo "copying packages"
echo "================================================"
${scripts}/docppackages ${arch} ${branch}
fi
fi
fi
endtime=$(date +%s)
echo "================================================"
echo "all done at $(date)"
echo "entire process took $(date -u -j -r $(($endtime - $starttime)) | awk '{print $4}')"
echo "================================================"