36ab4b3e9a
by optionally disabling progress report type output. This will make the daily pkgmastr mail a lot smaller without dropping useful information.
444 lines
14 KiB
Bash
Executable file
444 lines
14 KiB
Bash
Executable file
#!/bin/sh
|
|
#
|
|
# $NetBSD: binpkg-cache,v 1.21 2010/07/21 12:29:46 spz Exp $
|
|
#
|
|
# Script for generating a cache file with information about
|
|
# all binary packages contained in a directory.
|
|
#
|
|
# Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This code is derived from software contributed to The NetBSD Foundation
|
|
# by Dan McMahill.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
# 3. All advertising materials mentioning features or use of this software
|
|
# must display the following acknowledgement:
|
|
# This product includes software developed by the NetBSD
|
|
# Foundation, Inc. and its contributors.
|
|
# 4. Neither the name of The NetBSD Foundation nor the names of its
|
|
# contributors may be used to endorse or promote products derived
|
|
# from this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
|
|
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
|
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
|
|
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
# POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
|
|
TMPDIR=${TMPDIR:-/tmp}
|
|
PACKAGES=${PACKAGES:-/usr/pkgsrc/packages/}
|
|
AWK=${AWK:-awk}
|
|
CMP=${CMP:-cmp}
|
|
FIND=${FIND:-find}
|
|
GREP=${GREP:-grep}
|
|
GZIP_CMD=${GZIP_CMD:-gzip}
|
|
BZIP2=${BZIP2:-bzip2}
|
|
PKG_INFO=${PKG_INFO:-pkg_info}
|
|
PKG_SUFX=${PKG_SUFX:-.tgz}
|
|
SED=${SED:-sed}
|
|
SORT=${SORT:-sort}
|
|
STAT=${STAT:-stat}
|
|
|
|
cachefile=.pkgcache
|
|
summaryfile=pkg_summary
|
|
cacheversion=20050428
|
|
|
|
prompt="----> "
|
|
tab=" "
|
|
|
|
tmpd=${TMPDIR}/pkg-cache.$$
|
|
mkdir -m 0700 ${tmpd}
|
|
if test $? -ne 0 ; then
|
|
echo "ERROR: Could not create temporary directory ${tmpd}"
|
|
echo "Either you do not have write permission to ${tmpd} or"
|
|
echo "${tmpd} already exists"
|
|
exit 1
|
|
fi
|
|
all_dirs=${tmpd}/all_dirs
|
|
|
|
prog=$0
|
|
|
|
usage(){
|
|
echo "$prog - Generates cache files for each directory containing binary"
|
|
echo " packages. This cache file can then be used by the README.html"
|
|
echo " generation code to avoid having to call pkg_info(1) over and over"
|
|
echo " on the same binary package. In addition, pkg_summary.gz files for"
|
|
echo " use by the pkgtools/pkg_chk pacakge may be generated."
|
|
echo " "
|
|
echo "Usage: $prog [-d|--debug] [-v|--verbose] [-s|--summary] [-p|--packages <dir>]"
|
|
echo " "
|
|
echo " $prog -h|--help"
|
|
echo " "
|
|
echo " $prog -V|--version"
|
|
echo " "
|
|
echo "The options supported by $prog are: "
|
|
echo " "
|
|
echo " -d|--debug Enables debugging output"
|
|
echo " "
|
|
echo " -F|--force Forces a rebuild of the cache files even if they are"
|
|
echo " otherwise up to date."
|
|
echo " "
|
|
echo " -h|--help Displays this help message"
|
|
echo " "
|
|
echo " -p|--packages <dir> Specifies the top level directory to be searched"
|
|
echo " for binary packages."
|
|
echo " "
|
|
echo " -s|--summary Enables the creation of pkg_summary.gz files in each"
|
|
echo " directory containing binary packages. The pkg_summary.gz"
|
|
echo " file is used by the pkgtools/pkg_chk package."
|
|
echo " "
|
|
echo " -v|--verbose Enables verbose output from the script."
|
|
echo " "
|
|
echo " -V|--version Displays the version of this script and exits."
|
|
echo " "
|
|
echo "Returns 0 on success, 1 on errors, 2 if the packages"
|
|
echo "directory does not exist."
|
|
echo " "
|
|
echo "Example: $prog -v --packages /usr/pkgsrc/packages"
|
|
echo " "
|
|
}
|
|
|
|
clean_and_exit0(){
|
|
rm -fr ${tmpd}
|
|
exit 0
|
|
}
|
|
|
|
clean_and_exit1(){
|
|
rm -fr ${tmpd}
|
|
exit 1
|
|
}
|
|
|
|
clean_and_exit2(){
|
|
rm -fr ${tmpd}
|
|
exit 2
|
|
}
|
|
|
|
all_cache_files=""
|
|
|
|
#############################################################################
|
|
#
|
|
# process_binpkg_dir()
|
|
#
|
|
# Process a directory by checking to see if a cache file exists. If the
|
|
# cache file exists, make sure it is up to date. If the file does not
|
|
# exist, create one.
|
|
#
|
|
# also keep track of this directory so it can be added to the master
|
|
# cache.
|
|
#
|
|
|
|
process_binpkg_dir(){
|
|
rdir=`${GREP} "^${d} " ${all_dirs} | ${AWK} '{print $2}'`
|
|
need_update=no
|
|
if test -f ${d}/${cachefile} ; then
|
|
stale_entries=`${FIND} ${d} -type f -name \*${PKG_SUFX} -newer ${d}/${cachefile} -print`
|
|
|
|
# FIX_ME
|
|
#
|
|
# We also should find cache entries for files which no longer exist
|
|
# and nuke them. Right now we simply declare the entire cache out
|
|
# of date. Once we implement incremental updates to the cache,
|
|
# we need to remove the entries but not mark the entire cache as
|
|
# bad.
|
|
$if_debug echo " Checking for cache entries with no corresponding pkg."
|
|
# get the list of what pkgs belong in the cache
|
|
rm -f ${tmpd}/pkg_list ${tmpd}/cache_pkg_list
|
|
${FIND} ${d}/ -name \*${PKG_SUFX} -print | \
|
|
${SED} -e "s;^${d}/*;${rdir}/;g" -e 's;//;/;g' | \
|
|
${SORT} > ${tmpd}/pkg_list
|
|
|
|
# and get the list of what is in the cache
|
|
${AWK} '/pkgcache_begin/ {gsub(/pkgcache_begin[ \t]*/, ""); print}' \
|
|
${d}/${cachefile} | ${SORT} > ${tmpd}/cache_pkg_list
|
|
|
|
if ${CMP} -s ${tmpd}/pkg_list ${tmpd}/cache_pkg_list ; then
|
|
$if_debug echo " No extra cache entries in ${d}/${cachefile}"
|
|
else
|
|
$if_debug echo "Package list:"
|
|
$if_debug cat ${tmpd}/pkg_list
|
|
$if_debug echo "Cache list:"
|
|
$if_debug cat ${tmpd}/cache_pkg_list
|
|
echo " Entries found in ${d}/${cachefile} but no packages found"
|
|
need_update=yes
|
|
fi
|
|
else
|
|
stale_entries=""
|
|
fi
|
|
|
|
if test "X${force}" = "Xyes" -o "X${need_update}" = "Xyes" ; then
|
|
need_update=yes
|
|
echo "${tab}Forcing rebuild of cache ${d}/${cachefile}."
|
|
elif test ! -f ${d}/${cachefile} ; then
|
|
need_update=yes
|
|
echo "${tab}Missing cache file. ${d}/${cachefile} will be generated."
|
|
elif test -n "${stale_entries}" ; then
|
|
need_update=yes
|
|
echo "${tab}Stale cache file. ${d}/${cachefile} will be regenerated."
|
|
else
|
|
${GREP} "pkgcache_version ${cacheversion}" ${d}/${cachefile} >/dev/null 2>&1
|
|
if test $? -ne 0 ; then
|
|
need_update=yes
|
|
echo "${tab}Invalid version cache file. ${d}/${cachefile} will be regenerated."
|
|
echo "Need version ${cacheversion} but the file has"
|
|
${GREP} "^pkgcache_version " ${d}/${cachefile}
|
|
else
|
|
$if_verbose echo "${tab}Cache file ${d}/${cachefile} is up to date."
|
|
fi
|
|
fi
|
|
|
|
if test "${build_summary}" = "yes" -a ! -f ${d}/${summaryfile}.gz ; then
|
|
echo "${tab}Summary file ${d}/${summaryfile}.gz is missing and will be created."
|
|
need_update=yes
|
|
fi
|
|
if test "${build_summary}" = "yes" -a ${d}/${summaryfile}.gz -ot ${d}/${cachefile} ; then
|
|
echo "${tab}Summary file ${d}/${summaryfile}.gz is out of date and will be regenerated."
|
|
need_update=yes
|
|
fi
|
|
|
|
# FIX_ME
|
|
# We should use stale_entries in a way where we only update the
|
|
# cache file entries corresponding to these if we're rebuilding
|
|
# due to stale entries. That should save a good bit of time.
|
|
#
|
|
if test "X${need_update}" = "Xyes" ; then
|
|
echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
|
|
rm -f ${tmpd}/${summaryfile}
|
|
touch ${tmpd}/${summaryfile}
|
|
for f in ${d}/*${PKG_SUFX} ; do
|
|
fn=`grep "^${d} " ${all_dirs} | ${AWK} '{print $2}'`"/"`basename ${f}`
|
|
$if_debug echo " Adding ${fn} (${f}) to the cache"
|
|
echo " " >> ${tmpd}/${cachefile}
|
|
# stat(1) needs to be added to the bootstrap kit
|
|
# first if we want to use it here
|
|
#eval $(${STAT} -s ${f} 2>/dev/null)
|
|
echo "pkgcache_begin ${fn}" >> ${tmpd}/${cachefile}
|
|
#echo "pkgcache_mtime=${st_mtime}" >> ${tmpd}/${cachefile}
|
|
|
|
$if_debug echo "${PKG_INFO} -q -B ${f}"
|
|
${PKG_INFO} -q -B ${f} >> ${tmpd}/${cachefile}
|
|
|
|
if test "${build_summary}" = "yes" ; then
|
|
$if_debug echo "${PKG_INFO} -X ${f}"
|
|
${PKG_INFO} -X ${f} >> ${tmpd}/${summaryfile}
|
|
fi
|
|
echo "pkgcache_end ${fn}" >> ${tmpd}/${cachefile}
|
|
done
|
|
if test -f ${d}/${cachefile} ; then
|
|
rm -f ${d}/${cachefile}
|
|
fi
|
|
mv -f ${tmpd}/${cachefile} ${d}/${cachefile}
|
|
if test $? -ne 0 ; then
|
|
echo "********** WARNING **********"
|
|
echo "move of ${tmpd}/${cachefile} to ${d}/${cachefile} failed!"
|
|
echo "Perhaps you do not have write permissions to ${d}?"
|
|
echo "This directory will be dropped from the master cache file."
|
|
echo "********** WARNING **********"
|
|
return
|
|
fi
|
|
|
|
if test "${build_summary}" = "yes" ; then
|
|
if test -f ${d}/${summaryfile}.gz ; then
|
|
rm -f ${d}/${summaryfile}.gz
|
|
fi
|
|
cat ${tmpd}/${summaryfile} | ${GZIP_CMD} > ${d}/${summaryfile}.gz
|
|
if test $? -ne 0 ; then
|
|
echo "********** WARNING **********"
|
|
echo "${GZIP_CMD} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.gz failed!"
|
|
echo "Perhaps you do not have write permissions to ${d}?"
|
|
echo "********** WARNING **********"
|
|
return
|
|
fi
|
|
# if it's there, update it, otherwise don't bother
|
|
if test -f ${d}/${summaryfile}.bz2 ; then
|
|
rm -f ${d}/${summaryfile}.bz2
|
|
cat ${tmpd}/${summaryfile} | ${BZIP2} > ${d}/${summaryfile}.bz2
|
|
if test $? -ne 0 ; then
|
|
echo "********** WARNING **********"
|
|
echo "${BZIP2} of ${tmpd}/${summaryfile} to ${d}/${summaryfile}.bz2 failed!"
|
|
echo "Perhaps you do not have write permissions to ${d}?"
|
|
echo "********** WARNING **********"
|
|
return
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
fi
|
|
|
|
# if we got here, then this directory should have a good cache file in
|
|
# it and we should be able to add it to the master cache file
|
|
all_cache_files="${all_cache_files} ${d}/${cachefile}"
|
|
|
|
}
|
|
|
|
process_cache_files(){
|
|
echo "${prompt}Checking master cache file ${PACKAGES}/${cachefile}"
|
|
echo "pkgcache_version ${cacheversion}" > ${tmpd}/${cachefile}
|
|
if test -n "${all_cache_files}" ; then
|
|
for c in ${all_cache_files} ; do
|
|
echo "pkgcache_cachefile ${c}" >> ${tmpd}/${cachefile}
|
|
done
|
|
fi
|
|
if test ! -f ${PACKAGES}/${cachefile} ; then
|
|
echo "${tab}Creating master cache file ${PACKAGES}/${cachefile}"
|
|
mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
|
|
if test $? -ne 0 ; then
|
|
echo "********** ERROR **********"
|
|
echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
|
|
echo "Perhaps you do not have write permissions to ${PACKAGES}?"
|
|
echo "********** ERROR **********"
|
|
clean_and_exit1
|
|
fi
|
|
elif ${CMP} -s ${tmpd}/${cachefile} ${PACKAGES}/${cachefile} ; then
|
|
echo "${tab}Master cache file ${PACKAGES}/${cachefile} is up to date"
|
|
else
|
|
echo "${tab}Updating master cache file ${PACKAGES}/${cachefile}"
|
|
mv -f ${tmpd}/${cachefile} ${PACKAGES}/${cachefile}
|
|
if test $? -ne 0 ; then
|
|
echo "********** ERROR **********"
|
|
echo "move of ${tmpd}/${cachefile} to ${PACKAGES}/${cachefile} failed!"
|
|
echo "Perhaps you do not have write permissions to ${PACKAGES}?"
|
|
echo "********** ERROR **********"
|
|
clean_and_exit1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
######################################################################
|
|
#
|
|
# Handle command line options
|
|
#
|
|
######################################################################
|
|
|
|
if_debug=: # either ":" or ""
|
|
if_verbose=: # either ":" or ""
|
|
force=no
|
|
build_summary=no
|
|
|
|
while
|
|
test -n "$1"
|
|
do
|
|
case "$1" in
|
|
|
|
# Turn on debugging
|
|
-d|--debug)
|
|
if_debug=""
|
|
if_verbose=""
|
|
shift
|
|
;;
|
|
|
|
# Force a rebuilde of the cache
|
|
-F|--force)
|
|
force=yes
|
|
shift
|
|
;;
|
|
# Help
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
|
|
# Use the specified packages directory
|
|
-p|--packages)
|
|
PACKAGES=$2
|
|
shift 2
|
|
;;
|
|
|
|
# Build the pkg_summary.gz files
|
|
-s|--summary)
|
|
build_summary=yes
|
|
shift
|
|
;;
|
|
|
|
# Version
|
|
-V|--version)
|
|
${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s: Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
|
|
exit 0
|
|
;;
|
|
|
|
# Turn on verbose output, but not as noisy as debug
|
|
-v|--verbose)
|
|
if_verbose=""
|
|
shift
|
|
;;
|
|
|
|
-*) echo "$prog: ERROR: $1 is not a valid option"
|
|
usage
|
|
clean_and_exit1
|
|
;;
|
|
|
|
*)
|
|
break
|
|
;;
|
|
|
|
esac
|
|
done
|
|
|
|
if test $# -ne 0 ; then
|
|
echo "$0: $* is invalid"
|
|
usage
|
|
clean_and_exit1
|
|
fi
|
|
|
|
if test ! -d ${PACKAGES} ; then
|
|
echo "Packages directory ${PACKAGES} seems to be missing"
|
|
clean_and_exit2
|
|
fi
|
|
|
|
# put a trailing / after ${PACKAGES} in case ${PACKAGES} is
|
|
# a link.
|
|
|
|
# pass 1, we find all directories under PACKAGES. Note that this
|
|
# may contain some directories more than once depending on what sort
|
|
# of soft links may be in place
|
|
rm -f ${all_dirs}.tmp
|
|
for d in `${FIND} ${PACKAGES}/ -type d -follow -print` ; do
|
|
cname=`(cd ${d} && pwd -P)`
|
|
rname=`echo ${d} | ${SED} "s;^${PACKAGES}/*;;g"`
|
|
echo "${cname} ${rname}" >> ${all_dirs}.tmp
|
|
done
|
|
${SORT} -u -k1,1 ${all_dirs}.tmp > ${all_dirs}
|
|
$if_debug echo "Full directory list:"
|
|
$if_debug cat ${all_dirs}.tmp
|
|
$if_debug echo "Unique directory list:"
|
|
$if_debug cat ${all_dirs}
|
|
|
|
for d in `${AWK} '{print $1}' ${all_dirs}` ; do
|
|
$if_debug echo "${prompt}Processing directory ${d}"
|
|
is_pkg_dir=no
|
|
for f in ${d}/*${PKG_SUFX} ; do
|
|
if test -f "${f}" -a ! -h "${f}" ; then
|
|
${PKG_INFO} ${f} >/dev/null 2>&1
|
|
if test $? -eq 0 ; then
|
|
is_pkg_dir=yes
|
|
break
|
|
fi
|
|
fi
|
|
done
|
|
if test "X${is_pkg_dir}" = "Xyes" ; then
|
|
$if_verbose echo "${prompt}Checking cache in ${d}"
|
|
process_binpkg_dir
|
|
else
|
|
$if_debug echo "${prompt}no binary packages in ${d}"
|
|
fi
|
|
|
|
done
|
|
|
|
process_cache_files
|
|
|
|
clean_and_exit0
|