pkgsrc/mk/bulk/upload

#!/bin/sh
# $NetBSD: upload,v 1.48 2010/04/10 21:44:44 wiz Exp $

#
# Upload non-restricted binary pkgs to ftp server
#

AWK=${AWK:-/usr/bin/awk}
PKG_ADMIN="pkg_admin"
PKG_INFO="pkg_info"

set -eu

#
# Find out where we are
#
scriptdir=`dirname "$0"`
scriptdir=`cd "${scriptdir}" && pwd`

usage()
{
cat << EOF

$prog:  Uploads binary packages.

Usage:  $prog [-c|--config <file>] [-n|--no-upload] [-d|--debug] [-v|--verbose]
	$prog -h|--help
        $prog -V|--version

    -c | --config <file>
			Load the following configuration file instead of
			the default one.

    -d|--debug          Preserves some intermediate files to help
			debug any problems.

    -h|--help           Displays this help message and exits.

    -n|--no-upload      Does all of the work except for the actual upload.
			This option may be used to generate the upload script
			along with the list of packages to be excluded.

    -i|--no-install	Do not install the required packages; assume instead
			that they are already available.

    -v|--verbose        Enables verbose output.

    -V|--version        Displays the version of this script and exits.

Example:

    cd /usr/pkgsrc && $prog

Environment Variables:

    AWK  --  If specified, sets the awk program to be used

Files:

    pkgsrc/mk/bulk/build.conf

EOF
}

# print out error message and exit 1
die () {
	echo "$0: error:" 1>&2
	for i in "$@"; do
		echo "    $i" 1>&2
	done
	exit 1
}

prog=$0
debug=no
do_upload=yes
do_install=yes
verbose=no
while test $# -gt 0; do
	case "$1" in
	-c|--config)
		shift
		BULK_BUILD_CONF=$1; shift
		;;

	-d|--debug)
		debug=yes
		shift
		;;

	-h|--help)
		usage
		exit 0
		;;

	-i|--no-install)
		do_install=no
		shift
		;;

	-n|--no-upload)
		do_upload=no
		shift
		;;

	-v|--verbose)
		verbose=yes
		shift
		;;

	-V|--version)
		${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s:  Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
		exit 0
		;;

	-*)
		echo "$prog:  Unknown option:  $1" 1>&2
		usage
		exit 1
		;;

	*)
		break
		;;
	esac
done
if test "$#" -gt 0; then
	echo "$prog:  Unknown argument: $1" 1>&2
	usage
	exit 1
fi

install_required()
{

	[ "$do_install" = "yes" ] || return 0
	pkg=$1
	if [ "${verbose}" = "yes" ]; then
		echo "upload> Installing ${pkg}"
	fi
	(cd "$USR_PKGSRC" && cd "$pkg" && ${BMAKE} bulk-install) \
	|| {
		echo "upload> ERROR: Unable to install required package $pkg!" 1>&2
		echo "        Bailing out -- you're on your own." 1>&2
		exit 1
	}
}

MD5="digest md5";
SHA1="digest sha1";

_INFO_VER=`${PKG_INFO} -V`;
REQUIRED_PACKAGES="pkgtools/lintpkgsrc net/rsync"

opsys=`uname -s`
case "$opsys" in
NetBSD)	BMAKE=make;
	BSDSUM="sum -o 1";
	CKSUM="cksum";
	SYSVSUM="sum -o 2";
	;;
IRIX*)  BMAKE=bmake;
	BSDSUM="sum -r";
	CKSUM="cksum";
	SYSVSUM="sum";
	;;
*)	BMAKE=bmake;
	BSDSUM="";
	CKSUM="";
	SYSVSUM="";
	;;
esac

export BMAKE

# Bring in variables for bulk-install
BATCH=1
DEPENDS_TARGET=bulk-install
export BATCH DEPENDS_TARGET

#
# Find the configuration file.
#
BULK_BUILD_CONF="${BULK_BUILD_CONF-${scriptdir}/build.conf}"
case $BULK_BUILD_CONF in
	/*) ;;
	 *)     BULK_BUILD_CONF="${PWD}/${BULK_BUILD_CONF}"
esac

#
# Get the variables MAKECONF, RSYNC_DST, RSYNC_OPTS from the bulk build
# configuration file.
#
{ test -f "${BULK_BUILD_CONF}"		\
  && . "${BULK_BUILD_CONF}"		\
  && . "${scriptdir}/post-build-conf"	\
  && check_config_vars			\
  && export_config_vars
} || die "Cannot load config file ${BULK_BUILD_CONF}, aborting."

cd $USR_PKGSRC

if [ -z "$RSYNC_DST" ]; then
	echo "upload> ERROR: You must set the variable RSYNC_DST, see build.conf-example." 1>&2
	exit 1
fi

#
# Some temp files
#

umask 022
TMPDIR="${TMPDIR:-/tmp}"
TMP="${TMPDIR}"/pkg_upload.$$
(umask 077 && mkdir "${TMP}") \
|| {
        echo "upload> ERROR: cannot create temporary directory \"${TMP}\"." 1>&2
        exit 1
}

restricted_packages="$TMP/restricted_packages"
old_packages="$TMP/old_packages"
good_packages="$TMP/regular_packages"
all_good_packages="$TMP/all_regular_packages"

upload_general="$TMP"/upload_general

# May be different than $USR_PKGSRC:
echo "upload> Running ${BMAKE} to get the pkgsrc variables"
pkgsrcdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=_PKGSRCDIR`
packages=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=PACKAGES`
distdir=`cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=DISTDIR`
gzip_cmd=`cd pkgtools/lintpkgsrc; ${BMAKE} show-var VARNAME=GZIP_CMD USE_TOOLS=gzip`
pkg_info=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=PKG_INFO`
shell=`cd pkgtools/lintpkgsrc && ${BMAKE} show-var VARNAME=TOOLS_PLATFORM.sh`

# Pull in some pkgs needed
for pkg in ${REQUIRED_PACKAGES}; do
	install_required $pkg
done

case $LINTPKGSRC_CACHE in
yes|YES)
	lintpkgsrc_cache="-I `cd pkgtools/lintpkgsrc ; ${BMAKE} show-var VARNAME=LINTPKGSRC_DB`"
	;;
*)
	lintpkgsrc_cache=''
	;;
esac

echo "upload> Checking for out of date packages:"
# -p  =  report old versions of packages
lintpkgsrc $lintpkgsrc_cache -K $packages -P $pkgsrcdir -p > "${old_packages}.tmp"
sed 's@'$packages'/@@' < "${old_packages}.tmp" > "$old_packages"

RSFLAGS="-vap --progress $RSYNC_OPTS"

failed=no
cd $packages

echo "upload> Checking for restricted packages"
(cd All && env PKG_INFO="${pkg_info}" OUTDIR="${TMP}" \
	       ${shell} "${pkgsrcdir}/mk/bulk/sort-packages")

# Add the name of the package file, including all its symlinks to the
# list of files to be uploaded.
while read package; do
	ls -1 */"$package"
done < "$good_packages" > "$all_good_packages"

if [ "${MKSUMS}" = "yes" -o "${MKSUMS}" = "YES" ]; then

	echo "upload> Calculating checksum files..."

	SUMFILES="BSDSUM CKSUM MD5 SHA1 SYSVSUM"

	rm -f ${SUMFILES}

	if [ "${SIGN_AS-}" != "" ]; then
		install_required "security/gnupg"
		for i in ${SUMFILES}; do
			echo > $i
			echo "This file is signed with ${SIGN_AS}'s PGP key." >> $i
			echo >> $i
		done
	fi

	install_required "pkgtools/digest"

	[ -z "${BSDSUM}" ] && BSDSUM="echo"
	[ -z "${CKSUM}" ] && CKSUM="echo"
	[ -z "${SYSVSUM}" ] && SYSVSUM="echo"

	for pkg in `cat "${good_packages}"`; do
		pkg="All/$pkg"
		${BSDSUM}	"$pkg" >> BSDSUM
		${CKSUM}	"$pkg" >> CKSUM
		${MD5}		"$pkg" >> MD5
		${SHA1}		"$pkg" >> SHA1
		${SYSVSUM}	"$pkg" >> SYSVSUM
	done

	[ "${BSDSUM}" = "echo" ] && rm BSDSUM
	[ "${CKSUM}" = "echo" ] && rm CKSUM
	[ "${SYSVSUM}" = "echo" ] && rm SYSVSUM

	if [ "${SIGN_AS-}" != "" ]; then
		for i in ${SUMFILES}; do
			if [ -s $i ]; then
				echo "upload> Signing $i"
				gpg --clearsign $i && rm $i
			fi
		done
	else
		echo "upload> Checksum files not PGP-signed. Please do so manually!"
		echo "        (Run 'gpg --clearsign' on all of them)"
	fi
fi

if [ "${MKSUMMARY-}" = "yes" -o "${MKSUMMARY-}" = "YES" ]; then
	echo "upload> Creating summary file..."
	(cd "${packages}/All" \
		&& ls -t | grep '\.t[gb]z$' | while read n; do pkg_info -X "$n"; done) \
		| ${gzip_cmd} > "${packages}"/All/pkg_summary.gz
fi

cat <<EOF > "$upload_general"
#! /bin/sh
set -e
cd "$packages"
rsync $RSFLAGS --files-from="${all_good_packages}" --exclude-from="${old_packages}" . "$RSYNC_DST/"
EOF
chmod +x "$upload_general"

if [ "$do_upload" = "yes" ]; then
	echo "upload> Uploading packages"
	${shell} "$upload_general" \
	|| {
		echo "upload> ERROR: rsync failed.  To retry later, you can run $upload_general" 1>&2
		failed=yes
	}
else
	echo "upload> Skipping upload of packages."
	echo "        Run \"$upload_general\" to upload them later."
fi

# clean up temp files
if [ "$failed,$debug,$do_upload" = "no,no,yes" ]; then
	rm -fr "$TMP"
else
	echo "upload> Preserving temporary directory ${TMP}"
fi

if [ "$failed" = "yes" ]; then
	exit 1
fi