f8e206ffe7
(1) The script now assumes make(1) knows how to handle long dependency
chains properly. quickports is a list of ports that take a
long time to build by thesmelves (not ports that have long
dependency chains). The script adds several extra dependency
levels to the generate Makefile for those ports to make their
dependency chains longer.
(2) Use -R flag to cvs. Comment out the cvsup part.
(3) Preserve error messages for ports that didn't build the first
time around but did build on the retry.
(4) Record the package's timestamps in the log directory by doing
an "ls -asFlrt" in the packages/All directory.
(5) Add a little sleep after two background jobs so outputs won't
be garbled.
545 lines
16 KiB
Bash
Executable file
545 lines
16 KiB
Bash
Executable file
#!/bin/sh
|
|
|
|
# configurable variables
|
|
pb=/var/portbuild
|
|
|
|
. ${pb}/portbuild.conf
|
|
|
|
# packages for dependencies only
|
|
dummyports="x11/XFree86"
|
|
|
|
# packages that take very long to build -- try to start building these first
|
|
quickports="lang/ghc games/civ2demo games/rt2-demo x11/XFree86-4 editors/openoffice"
|
|
|
|
status=${pb}/status
|
|
|
|
errorexit () {
|
|
echo "$1" > ${status}
|
|
exit $1
|
|
}
|
|
|
|
usage () {
|
|
echo "usage: [-nobuild] [-noindex] [-noduds] [-nocvsup] [-nocvs] [-norestr] [-noplistcheck] [-nodummy] [-ftp] branch date"
|
|
errorexit 1
|
|
}
|
|
|
|
# usage: makeindex pb scripts branch user
|
|
makeindex () {
|
|
pb=$1
|
|
scripts=$2
|
|
branch=$3
|
|
user=$4
|
|
|
|
cd ${pb}/usr/ports
|
|
echo "================================================"
|
|
echo "generating index"
|
|
echo "================================================"
|
|
echo "index generation started at $(date)"
|
|
${scripts}/makeindex ${branch} || errorexit 1
|
|
echo "index generation ended at $(date)"
|
|
echo $(wc -l <INDEX) "lines in INDEX"
|
|
chown ${user} INDEX
|
|
cp -p INDEX INDEX-${branch}
|
|
}
|
|
|
|
# usage: checkindex pb
|
|
checkindex () {
|
|
pb=$1
|
|
|
|
cd ${pb}/usr/ports
|
|
if grep -q non-existent INDEX; then
|
|
echo "errors in INDEX:"
|
|
grep -n non-existent INDEX
|
|
errorexit 1
|
|
# grep -C non-existent INDEX
|
|
# grep -v non-existent INDEX > INDEX.tmp
|
|
# mv -f INDEX.tmp INDEX
|
|
fi
|
|
if ! awk -F '|' '{if (NF != 10) { error=1; printf("line %d: %s\n", NR, $0)}} END {if (error == 1) exit(1)}' INDEX; then
|
|
echo "error in INDEX"
|
|
errorexit 1
|
|
fi
|
|
}
|
|
|
|
# usage: makeduds pb scripts branch
|
|
makeduds () {
|
|
pb=$1
|
|
scripts=$2
|
|
branch=$3
|
|
|
|
cd ${pb}/usr/ports
|
|
echo "================================================"
|
|
echo "generating duds"
|
|
echo "================================================"
|
|
echo "duds generation started at $(date)"
|
|
if ! ${scripts}/makeduds ${branch}; then
|
|
echo "error(s) detected, exiting script at $(date)"
|
|
errorexit 1
|
|
fi
|
|
echo "duds generation ended at $(date)"
|
|
echo $(wc -l < ${pb}/${branch}/duds) "items in duds"
|
|
echo "duds diff:"
|
|
diff ${pb}/${branch}/duds.old ${pb}/${branch}/duds
|
|
cp -p ${pb}/${branch}/duds ${pb}/${branch}/duds.old
|
|
}
|
|
|
|
# usage: setupnode pb scripts branch me node md5
|
|
setupnode () {
|
|
pb=$1
|
|
scripts=$2
|
|
branch=$3
|
|
me=$4
|
|
node=$5
|
|
md5=$6
|
|
|
|
echo "setting up of $node started at $(date)"
|
|
ssh -n ${node} mkdir -p ${scripts}
|
|
scp -p ${scripts}/setupnode ${node}:${scripts}
|
|
ssh -n ${node} ${scripts}/setupnode ${me} ${pb} ${branch} ${md5}
|
|
echo "setting up of $node ended at $(date)"
|
|
}
|
|
|
|
# usage: restrictedlist pb scripts branch
|
|
restrictedlist () {
|
|
pb=$1
|
|
scripts=$2
|
|
branch=$3
|
|
|
|
cd ${pb}/usr/ports
|
|
echo "================================================"
|
|
echo "creating restricted list"
|
|
echo "================================================"
|
|
echo "restricted list generation started at $(date)"
|
|
make ECHO_MSG=/usr/bin/true clean-restricted-list \
|
|
| sed -e "s./usr/ports/distfiles/./distfiles/.g" \
|
|
-e "s./usr/ports/./${branch}/.g" \
|
|
> ${pb}/${branch}/restricted.sh
|
|
echo "restricted list generation ended at $(date)"
|
|
echo $(grep -c '^#' ${pb}/${branch}/restricted.sh) "ports in ${pb}/${branch}/restricted.sh"
|
|
}
|
|
|
|
# usage: cdromlist pb scripts branch
|
|
cdromlist () {
|
|
pb=$1
|
|
scripts=$2
|
|
branch=$3
|
|
|
|
cd ${pb}/usr/ports
|
|
echo "================================================"
|
|
echo "creating cdrom list"
|
|
echo "================================================"
|
|
echo "cdrom list generation started at $(date)"
|
|
make ECHO_MSG=/usr/bin/true clean-for-cdrom-list \
|
|
| sed -e "s./usr/ports/distfiles/./distfiles/.g" \
|
|
-e "s./usr/ports/./${branch}/.g" \
|
|
> ${pb}/${branch}/cdrom.sh
|
|
echo "cdrom list generation ended at $(date)"
|
|
echo $(grep -c '^#' ${pb}/${branch}/cdrom.sh) "ports in ${pb}/${branch}/cdrom.sh"
|
|
}
|
|
|
|
# usage: archiveports pb branch
|
|
archiveports () {
|
|
pb=$1
|
|
branch=$2
|
|
|
|
echo "started archive of /usr/ports at $(date)"
|
|
cd ${pb}/usr
|
|
tar --exclude CVS -czf ${pb}/${branch}/tarballs/ports.tar.gz ports
|
|
echo "ended archive of /usr/ports at $(date)"
|
|
}
|
|
|
|
# usage: generatemd5 pb branch
|
|
generatemd5 () {
|
|
pb=$1
|
|
branch=$2
|
|
|
|
echo "started generating CHECKSUM.MD5 at $(date)"
|
|
cd ${pb}/${branch}/packages/All
|
|
find . -name '*.tgz' | sort | sed -e 's/^..//' | xargs md5 > CHECKSUM.MD5
|
|
echo "ended generating CHECKSUM.MD5 at $(date)"
|
|
}
|
|
|
|
scripts=${pb}/scripts
|
|
|
|
umask 002
|
|
export PORTSDIR=${pb}/usr/ports
|
|
|
|
me=$(hostname -s)
|
|
|
|
echo "Subject: $me package building logs"
|
|
echo
|
|
echo "Called with arguments: "${1+"$@"}
|
|
echo "Started at $(date)"
|
|
|
|
starttime=$(date +%s)
|
|
|
|
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/X11R6/bin
|
|
|
|
if [ $# = 0 ]; then
|
|
usage
|
|
fi
|
|
|
|
nobuild=0
|
|
noindex=0
|
|
noduds=0
|
|
nocvsup=0
|
|
nocvs=0
|
|
norestr=0
|
|
noplistcheck=0
|
|
nodummy=0
|
|
ftp=0
|
|
|
|
# optional arguments
|
|
while [ $# -gt 2 ]; do
|
|
case "x$1" in
|
|
x-nobuild)
|
|
nobuild=1
|
|
;;
|
|
x-noindex)
|
|
noindex=1
|
|
;;
|
|
x-noduds)
|
|
noduds=1
|
|
;;
|
|
x-nocvsup)
|
|
nocvsup=1
|
|
;;
|
|
x-nocvs)
|
|
nocvs=1
|
|
;;
|
|
x-norestr)
|
|
norestr=1
|
|
;;
|
|
x-noplistcheck)
|
|
noplistcheck=1
|
|
;;
|
|
x-nodummy)
|
|
nodummy=1
|
|
;;
|
|
x-ftp)
|
|
ftp=1
|
|
;;
|
|
*)
|
|
usage
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# mandatory arguments
|
|
branch=$1
|
|
date=$2
|
|
if [ "x$branch" != x3 -a "x$branch" != x4 -a "x$branch" != x5 ]; then
|
|
usage
|
|
fi
|
|
|
|
if [ "$norestr" = 1 ]; then
|
|
export NO_RESTRICTED=t
|
|
fi
|
|
|
|
if [ "$noplistcheck" = 1 ]; then
|
|
export NOPLISTCHECK=t
|
|
fi
|
|
|
|
if [ "$nodummy" = 1 ]; then
|
|
export NODUMMY=t
|
|
export XFREE86_VERSION=4
|
|
fi
|
|
|
|
cd ${pb}/usr/ports
|
|
if [ "$nocvsup" = 0 ]; then
|
|
# echo "================================================"
|
|
# echo "running cvsup"
|
|
# echo "================================================"
|
|
# su ${user} -c 'cvsup -g -L 0 /etc/supfile.cvsup'
|
|
echo "================================================"
|
|
echo "running cvs update on /usr/ports"
|
|
echo "================================================"
|
|
su ${user} -c 'cvs -qR update -d -P'
|
|
date > ${pb}/cvsdone
|
|
echo "================================================"
|
|
echo "running make checksubdirs"
|
|
echo "================================================"
|
|
make checksubdirs
|
|
echo "================================================"
|
|
echo "running cvs update on /usr/opt/doc"
|
|
echo "================================================"
|
|
cd ${pb}/usr/opt/doc
|
|
su ${user} -c 'cvs -qR update -d -P'
|
|
fi
|
|
|
|
if [ "$nocvs" = 0 ]; then
|
|
echo "================================================"
|
|
echo "running cvs update on /${branch}/src"
|
|
echo "================================================"
|
|
cd ${pb}/${branch}/src
|
|
su ${user} -c 'cvs -qR update -d -P'
|
|
fi
|
|
|
|
# this one not in background to check return status
|
|
|
|
if [ "$noduds" = 0 ]; then
|
|
makeduds ${pb} ${scripts} ${branch}
|
|
fi
|
|
|
|
if [ "$noindex" = 0 ]; then
|
|
makeindex ${pb} ${scripts} ${branch} ${user} &
|
|
fi
|
|
|
|
md5=$(/sbin/md5 ${pb}/${branch}/tarballs/bindist.tar | awk '{print $4}')
|
|
echo "================================================"
|
|
echo "setting up nodes"
|
|
echo "================================================"
|
|
for node in $(awk '{print $1}' ${pb}/mlist); do
|
|
setupnode ${pb} ${scripts} ${branch} ${me} ${node} ${md5} &
|
|
sleep 2
|
|
done
|
|
|
|
restrictedlist ${pb} ${scripts} ${branch} &
|
|
sleep 2
|
|
|
|
cdromlist ${pb} ${scripts} ${branch} &
|
|
|
|
wait
|
|
|
|
checkindex ${pb}
|
|
|
|
echo "all preparation ended at $(date)"
|
|
|
|
cd ${pb}/usr/ports
|
|
make parallel branch=${branch} > ../../${branch}/Makefile
|
|
|
|
cd ${pb}/usr/ports
|
|
|
|
if [ "$nodummy" = 0 ]; then
|
|
dummypkgs=""
|
|
for i in ${dummyports}; do
|
|
if [ -d $i ]; then
|
|
dummypkgs="${dummypkgs} $(cd $i; make package-name).tgz"
|
|
else
|
|
echo "dummy port directory \"$i\" does not exist -- skipping"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# hack to extend length of dependency chain to build quickports first
|
|
for i in ${quickports}; do
|
|
if [ -d $i ]; then
|
|
quickpkg="$(cd $i; make package-name).tgz"
|
|
echo "all: dummy1-$quickpkg" >> ../../${branch}/Makefile
|
|
echo "dummy1-$quickpkg: dummy2-$quickpkg" >> ../../${branch}/Makefile
|
|
echo "dummy2-$quickpkg: dummy3-$quickpkg" >> ../../${branch}/Makefile
|
|
echo "dummy3-$quickpkg: $quickpkg" >> ../../${branch}/Makefile
|
|
else
|
|
echo "quick port directory \"$i\" does not exist -- skipping"
|
|
fi
|
|
done
|
|
|
|
cd ${pb}
|
|
if [ "$nobuild" = 0 ]; then
|
|
### rm -rf bak/distfiles
|
|
### mv -f distfiles bak
|
|
mkdir -p distfiles
|
|
chown -R ${user} distfiles
|
|
cd ${pb}/${branch}/bak
|
|
rm -rf errors logs packages old-errors
|
|
cd ${pb}/${branch}
|
|
mv -f errors logs packages old-errors make.* bak
|
|
###delete old packages -- use when diskspace is scarce
|
|
# if cd bak/packages/All; then
|
|
# for i in *.tgz; do
|
|
# cp /dev/null $i
|
|
# done
|
|
# fi
|
|
shortdate=$(echo ${date} | sed -e 's/..$//')
|
|
cd ${pb}/${branch}
|
|
mkdir -p packages/All
|
|
mkdir -p ${pb}/archive/errorlogs
|
|
rm -rf ${pb}/archive/errorlogs/e.${branch}.${date} ${pb}/archive/errorlogs/e.${branch}.${shortdate}
|
|
mkdir -p ${pb}/archive/errorlogs/e.${branch}.${date}
|
|
ln -sf ${pb}/archive/errorlogs/e.${branch}.${date} ${pb}/${branch}/errors
|
|
ln -sf e.${branch}.${date} ${pb}/archive/errorlogs/e.${branch}.${shortdate}
|
|
mkdir -p ${pb}/${branch}/errors/old-errors
|
|
rm -rf ${pb}/archive/errorlogs/a.${branch}.${date} ${pb}/archive/errorlogs/a.${branch}.${shortdate}
|
|
mkdir -p ${pb}/archive/errorlogs/a.${branch}.${date}
|
|
ln -sf ${pb}/archive/errorlogs/a.${branch}.${date} ${pb}/${branch}/logs
|
|
ln -sf a.${branch}.${date} ${pb}/archive/errorlogs/a.${branch}.${shortdate}
|
|
cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/e.${branch}.${date}/cvsdone
|
|
cp -p ${pb}/cvsdone ${pb}/archive/errorlogs/a.${branch}.${date}/cvsdone
|
|
cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/e.${branch}.${date}/INDEX
|
|
cp -p ${pb}/usr/ports/INDEX-${branch} ${pb}/archive/errorlogs/a.${branch}.${date}/INDEX
|
|
chown -RL ${user} errors logs packages
|
|
echo "error logs in ${pb}/archive/errorlogs/e.${branch}.${date}"
|
|
if [ "$nodummy" = 0 ]; then
|
|
for dir in ${dummyports}; do
|
|
cp -p tarballs/$(cd ${pb}/usr/ports/$dir; make package-name).tgz packages/All
|
|
done
|
|
fi
|
|
|
|
find /tmp -name 'make*' -mmin +60 | xargs rm -f
|
|
|
|
count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist | sed -e 's/\..*$//')
|
|
cd ${pb}/${branch}/packages/All
|
|
ln -sf ../../Makefile .
|
|
echo "================================================"
|
|
echo "building packages (phase 1)"
|
|
echo "================================================"
|
|
echo "started at $(date)"
|
|
phase1start=$(date +%s)
|
|
make -k -j$count all > ../../make.0 2>&1 </dev/null
|
|
echo "ended at $(date)"
|
|
phase1end=$(date +%s)
|
|
echo "phase 1 took $(date -u -j -r $(($phase1end - $phase1start)) | awk '{print $4}')"
|
|
echo $(echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) - 2 | bc) "packages built"
|
|
echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
|
|
echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
|
|
cd ${pb}/${branch}
|
|
if grep -qE '(ptimeout|pnohang): killing' make.0; then
|
|
echo "The following port(s) timed out:"
|
|
grep -E '(ptimeout|pnohang): killing' make.0 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/'
|
|
fi
|
|
ls -asFlrt ${pb}/${branch}/packages/All > ${pb}/${branch}/logs/ls-lrt-1
|
|
cp -rp errors old-errors
|
|
cd ${pb}/${branch}/old-errors
|
|
${pb}/scripts/processlogs
|
|
|
|
md5=$(/sbin/md5 ${pb}/${branch}/tarballs/bindist.tar | awk '{print $4}')
|
|
echo "================================================"
|
|
echo "setting up nodes"
|
|
echo "================================================"
|
|
for node in $(awk '{print $1}' ${pb}/mlist); do
|
|
setupnode ${pb} ${scripts} ${branch} ${me} ${node} ${md5} &
|
|
sleep 2
|
|
done
|
|
|
|
wait
|
|
|
|
echo "setting up of nodes ended at $(date)"
|
|
|
|
count=$(awk '{sum+=$2}END{print sum+NR/2}' ${pb}/mlist | sed -e 's/\..*$//')
|
|
cd ${pb}/${branch}/packages/All
|
|
echo "================================================"
|
|
echo "building packages (phase 2)"
|
|
echo "================================================"
|
|
echo "started at $(date)"
|
|
phase2start=$(date +%s)
|
|
make -k -j$count all > ../../make.1 2>&1 </dev/null
|
|
echo "ended at $(date)"
|
|
phase2end=$(date +%s)
|
|
echo "phase 2 took $(date -u -j -r $(($phase2end - $phase2start)) | awk '{print $4}')"
|
|
rm Makefile
|
|
if [ "$nodummy" = 0 ]; then
|
|
rm ${dummypkgs}
|
|
fi
|
|
${pb}/scripts/chopindex ${pb}/usr/ports/INDEX-${branch} ${pb}/${branch}/packages > \
|
|
${pb}/${branch}/packages/INDEX
|
|
echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
|
|
echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
|
|
echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
|
|
echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
|
|
if [ "$norestr" = 0 ]; then
|
|
echo "deleting restricted ports"
|
|
sh ${pb}/${branch}/restricted.sh
|
|
${pb}/scripts/chopindex ${pb}/usr/ports/INDEX ${pb}/${branch}/packages > \
|
|
${pb}/${branch}/packages/INDEX
|
|
echo $(ls -1 ${pb}/${branch}/packages/All | wc -l) "packages built"
|
|
echo $(cat ${pb}/${branch}/packages/INDEX | wc -l) "lines in INDEX"
|
|
echo $(echo $(du -sk ${pb}/${branch}/packages | awk '{print $1}') / 1024 | bc) "MB of packages"
|
|
echo $(echo $(du -sk ${pb}/distfiles | awk '{print $1}') / 1024 | bc) "MB of distfiles"
|
|
fi
|
|
rm -rf ${pb}/${branch}/bad
|
|
mkdir -p ${pb}/${branch}/bad
|
|
echo "checking packages"
|
|
for i in *.tgz; do
|
|
if ! gzip -t $i; then
|
|
echo "Warning: package $i is bad, moving to ${pb}/${branch}/bad"
|
|
# the latest link will be left behind...
|
|
mv $i ${pb}/${branch}/bad
|
|
rm ../*/$i
|
|
fi
|
|
done
|
|
cd ${pb}/${branch}
|
|
if grep -qE '(ptimeout|pnohang): killing' make.1; then
|
|
echo "The following port(s) timed out:"
|
|
grep -E '(ptimeout|pnohang): killing' make.1 | sed -e 's/^.*ptimeout:/ptimeout:/' -e 's/^.*pnohang:/pnohang:/'
|
|
fi
|
|
cd ${pb}/${branch}/old-errors
|
|
new=""
|
|
for i in *.log; do
|
|
if [ ! -f ../errors/$i ]; then
|
|
new="$new $(basename $i .log)"
|
|
cp -p ${i} ${pb}/${branch}/errors/old-errors
|
|
fi
|
|
done
|
|
if [ "x$new" != "x" ]; then
|
|
echo "The following port(s) didn't build the first time around: $new"
|
|
fi
|
|
archiveports ${pb} ${branch} &
|
|
generatemd5 ${pb} ${branch} &
|
|
sleep 5
|
|
echo "================================================"
|
|
echo "new failures"
|
|
echo "================================================"
|
|
cd ${pb}/${branch}/errors
|
|
for i in *.log; do
|
|
if [ ! -f ${pb}/${branch}/bak/errors/$i ]; then
|
|
echo -n " $(basename $i .log)"
|
|
fi
|
|
done
|
|
if cd ${pb}/${branch}/bak/packages/All; then
|
|
echo
|
|
echo "================================================"
|
|
echo "old packages"
|
|
echo "================================================"
|
|
for i in *.tgz; do
|
|
if [ ! -f ${pb}/${branch}/packages/All/$i ]; then
|
|
echo -n " $(basename $i .tgz)"
|
|
fi
|
|
done
|
|
fi
|
|
if cd ${pb}/${branch}/bak/errors; then
|
|
echo
|
|
echo "================================================"
|
|
echo "old failures"
|
|
echo "================================================"
|
|
for i in *.log; do
|
|
if [ ! -f ${pb}/${branch}/errors/$i ]; then
|
|
echo -n " $(basename $i .log)"
|
|
fi
|
|
done
|
|
fi
|
|
echo
|
|
echo "================================================"
|
|
echo "new packages"
|
|
echo "================================================"
|
|
cd ${pb}/${branch}/packages/All
|
|
for i in *.tgz; do
|
|
if [ ! -f ${pb}/${branch}/bak/packages/All/$i ]; then
|
|
echo -n " $(basename $i .tgz)"
|
|
fi
|
|
done
|
|
echo
|
|
wait
|
|
ls -asFlrt ${pb}/${branch}/packages/All > ${pb}/${branch}/logs/ls-lrt
|
|
cp -p ${pb}/${branch}/make.[01] ${pb}/${branch}/logs
|
|
echo "================================================"
|
|
echo "copying distfiles"
|
|
echo "================================================"
|
|
echo "started at $(date)"
|
|
cd ${pb}
|
|
rm -rf bak/distfiles
|
|
mv -f distfiles bak
|
|
su ${user} -c ${scripts}/cpdistfiles > ${pb}/cpdistfiles.log 2>&1 </dev/null &
|
|
if [ "$ftp" = 1 ]; then
|
|
echo "ended at $(date)"
|
|
echo "================================================"
|
|
echo "copying packages"
|
|
echo "================================================"
|
|
su ${user} -c "${scripts}/docppackages ${branch}"
|
|
fi
|
|
fi
|
|
endtime=$(date +%s)
|
|
echo "================================================"
|
|
echo "all done at $(date)"
|
|
echo "entire process took $(date -u -j -r $(($endtime - $starttime)) | awk '{print $4}')"
|
|
echo "================================================"
|