freebsd-ports/Tools/portbuild/scripts/pdispatch

#!/bin/sh
#
# pdispatch <arch> <branch> <command> <package.tgz> [<args> ...]
#
# Choose a random machine from ${buildroot}/ulist and dispatch the
# job to it via the ptimeout script.

pb=/var/portbuild
arch=$1
shift
. ${pb}/${arch}/portbuild.conf
. ${pb}/scripts/buildenv

# wait 100 hours maximum
timeout=360000
# number of lines of log to send
loglength=1000
hdrlength=6

branch=$1
command=$2
shift 2

buildenv ${pb} ${arch} ${branch}

# ssh -x doesn't work on some machines
unset DISPLAY

pkgname=$(basename $1 ${PKGSUFFIX})

if grep -qxF ${pkgname} ${pb}/${arch}/${branch}/duds; then
  echo "skipping ${pkgname}"
  exit 1
fi

if [ -z "${pkgname}" ]; then
  echo "null packagename"
  exit 1
fi

args=${1+"$@"}
flags=""
clean=1
if [ "x$NOCLEAN" != "x" ]; then
  flags="${flags} -noclean"
  clean=0
fi
if [ "x$NO_RESTRICTED" != "x" ]; then
  flags="${flags} -norestr"
fi
if [ "x$NOPLISTCHECK" != "x" ]; then
  flags="${flags} -noplistcheck"
fi
if [ "x$WANT_DISTFILES" != "x" ]; then
  flags="${flags} -distfiles"
fi
if [ "x$FETCH_ORIGINAL" != "x" ]; then
  flags="${flags} -fetch-original"
fi
if [ "x$TRYBROKEN" != "x" ]; then
  flags="${flags} -trybroken"
fi

while `true`; do
  host=
  chroot=
  while [ -z "${host}" -o -z "${chroot}" ]; do
    chroot=
    host=$(lockf -k ${pb}/${arch}/queue/.lock ${pb}/scripts/getmachine ${pb} ${arch} ${branch})
    # If ulist is empty, then all build machines are busy, so try again in 15 seconds.
    if [ -z "${host}" ]; then
      sleep 15
    else
      . ${pb}/${arch}/portbuild.conf
      test -f ${pb}/${arch}/portbuild.${host} && . ${pb}/${arch}/portbuild.${host}
      chroot=$(ssh -a -n ${client_user}@${host} ${sudo_cmd} ${pb}/scripts/claim-chroot ${arch} ${branch} ${pkgname})
      if [ -z "${chroot}" ]; then
        echo "Failed to claim chroot on ${host}"
        lockf -k ${pb}/${arch}/queue/.lock ${pb}/scripts/releasemachine ${arch} ${host}
      fi
    fi
  done
  
  . ${pb}/${arch}/portbuild.conf
  test -f ${pb}/${arch}/portbuild.${host} && . ${pb}/${arch}/portbuild.${host}
  
  rm -f ${pb}/${arch}/${branch}/logs/${pkgname}.log ${pb}/${arch}/${branch}/logs/${pkgname}.log.bz2
  rm -f ${pb}/${arch}/${branch}/errors/${pkgname}.log ${pb}/${arch}/${branch}/errors/${pkgname}.log.bz2

  echo "dispatching: ssh -a -t -n ${client_user}@${host} ${sudo_cmd} ${command} ${arch} ${branch} ${chroot} ${flags} \"$ED\" \"$PD\" \"$FD\" \"$BD\" \"$RD\" ${args}"
  ${pb}/scripts/ptimeout.host $timeout ssh -a -t -n ${client_user}@${host} ${sudo_cmd} ${command} ${arch} ${branch} ${chroot} ${flags} \"$ED\" \"$PD\" \"$FD\" \"$BD\" \"$RD\" ${args} 2>&1
  error=$?

  #if grep -q " failed unexpectedly on " ${pb}/${arch}/${branch}/logs/${pkgname}.pre.log; then
  #  cat ${pb}/${arch}/${branch}/logs/${pkgname}.pre.log | mail -s "${pkgname} failed uncleanly on ${arch} ${branch}" ${mailto}
  #else
  #  rm ${pb}/${arch}/${branch}/logs/${pkgname}.pre.log
  #fi
  
  # Pull in the results of the build from the client
  
  scp ${client_user}@${host}:${chroot}/tmp/${pkgname}.log ${pb}/${arch}/${branch}/logs/${pkgname}.log
  (ssh -a -n ${client_user}@${host} test -f ${chroot}/tmp/work.tbz ) && scp ${client_user}@${host}:${chroot}/tmp/work.tbz ${pb}/${arch}/${branch}/wrkdirs/${pkgname}.tbz
  
  # XXX Set dirty flag if any of the scp's fail

  mkdir -p ${pb}/${arch}/${branch}/distfiles/.pbtmp/${pkgname}
  ssh -a -n ${client_user}@${host} tar -C ${chroot}/tmp/distfiles -cf - . | \
    tar --unlink -C ${pb}/${arch}/${branch}/distfiles/.pbtmp/${pkgname} -xvf -
  touch ${pb}/${arch}/${branch}/distfiles/.pbtmp/${pkgname}/.done
  
  if [ "${error}" = 0 ]; then
    ssh -a -n ${client_user}@${host} tar -C ${chroot}/tmp -cf - packages | \
      tar --unlink -C ${pb}/${arch}/${branch} -xvf -
    test -f ${pb}/${arch}/${branch}/packages/All/${pkgname}${PKGSUFFIX} && \
      touch ${pb}/${arch}/${branch}/packages/All/${pkgname}${PKGSUFFIX}
    rm -f ${pb}/${arch}/${branch}/errors/${pkgname}.log && \
      touch ${pb}/${arch}/${branch}/errors/.force
    lockf -k ${pb}/${arch}/${branch}/failure.lock ${pb}/scripts/buildsuccess ${arch} ${branch} ${pkgname}
    log=${pb}/${arch}/${branch}/logs/$pkgname.log
    if grep -q "even though it is marked BROKEN" ${log}; then
      echo | mail -s "${pkgname} BROKEN but built on ${arch} ${branch}" ${mailto}
    fi
    if grep -q "^list of .*file" ${log}; then
      buildlogdir=$(realpath ${pb}/${arch}/${branch}/logs/)
      baselogdir=$(basename ${buildlogdir})
      (sed -e '/^build started/,$d' $log;echo;echo "For the full build log, see"; echo; echo "  http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;sed -e '1,/^=== Checking filesystem state/d' $log) | mail -s "${pkgname} pkg-plist errors on ${arch} ${branch}" ${mailto}
    fi
  else
    log=${pb}/${arch}/${branch}/errors/${pkgname}.log
    scp ${client_user}@${host}:${chroot}/tmp/${pkgname}.log ${log} || (echo ${chroot}@${host}; ssh -a -n ${client_user}@${host} ls -laR ${chroot}/tmp) | mail -s "${pkgname} logfile not found" ${mailto}
    if ! grep -q "even though it is marked BROKEN" ${log}; then
      buildlogdir=$(realpath ${pb}/${arch}/${branch}/logs/)
      baselogdir=$(basename ${buildlogdir})
      if [ `wc -l ${log} | awk '{print $1}'` -le `expr ${loglength} + ${hdrlength}` ]; then
        (echo "You can also find this build log at"; echo; echo "  http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;cat ${log}) | mail -s "${pkgname} failed on ${arch} ${branch}" ${mailto}
      else
        (echo "Excerpt from the build log at"; echo; echo "  http://${master}/errorlogs/${arch}-errorlogs/${baselogdir}/$(basename $log)";echo;sed -e '/^build started/,$d' $log;echo;echo "  [... lines trimmed ...]";echo;tail -${loglength} ${log}) | mail -s "${pkgname} failed on ${arch} ${branch}" ${mailto}
      fi
    fi
    lockf -k ${pb}/${arch}/${branch}/failure.lock ${pb}/scripts/buildfailure ${arch} ${branch} ${pkgname}
  fi
  
  ssh -a -n ${client_user}@${host} ${sudo_cmd} ${pb}/scripts/clean-chroot ${arch} ${branch} ${chroot} ${clean}
  
  lockf -k ${pb}/${arch}/queue/.lock ${pb}/scripts/releasemachine ${arch} ${host}

  # XXX Set a dirty variable earlier and check here
  if grep -q "^build of .*ended at" ${pb}/${arch}/${branch}/logs/${pkgname}.log; then
    exit ${error}
  else
    echo "Build of ${pkgname} in ${host}:/${chroot} failed uncleanly, rebuilding"
    sleep 120
  fi
done