490 lines
13 KiB
Bash
Executable file
490 lines
13 KiB
Bash
Executable file
#!/bin/sh
|
|
# $NetBSD: mkreadme,v 1.33 2021/04/05 10:32:38 nia Exp $
|
|
#
|
|
# Script for index.html generation
|
|
#
|
|
# Copyright (c) 2002, 2005 The NetBSD Foundation, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This code is derived from software contributed to The NetBSD Foundation
|
|
# by Dan McMahill.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
|
|
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
|
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
|
|
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
# POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
|
|
TMPDIR=${TMPDIR:-/tmp/mkreadme}
|
|
PKGSRCDIR=${PKGSRCDIR:-/usr/pkgsrc}
|
|
AWK=${AWK:-/usr/bin/awk}
|
|
|
|
opsys=`uname -s`
|
|
case "$opsys" in
|
|
*BSD)
|
|
makeprog=make
|
|
;;
|
|
|
|
*)
|
|
makeprog=bmake
|
|
;;
|
|
esac
|
|
|
|
BMAKE=${BMAKE:-${makeprog}}
|
|
|
|
usage(){
|
|
echo "$prog - Generates index.html files for a pkgsrc tree"
|
|
echo "Usage: $prog [-c|--cdrom] [-C|--prune] [-d|--debug] [-f|--ftp] "
|
|
echo " [-q|--quiet] "
|
|
echo " [-p|--pkgsrc directory] "
|
|
echo " [-P|--packages directory] [-r|--restart] "
|
|
echo " [-s|--summary]"
|
|
echo " [-S|--save-database]"
|
|
echo " "
|
|
echo " $prog -h|--help"
|
|
echo " "
|
|
echo " $prog -v|--version"
|
|
echo " "
|
|
echo "The options supported by $prog are: "
|
|
echo " "
|
|
echo " -C|--prune Prune unused index.html files which may exist in"
|
|
echo " pkg directories which have been removed from CVS."
|
|
echo " "
|
|
echo " -c|--cdrom Generates CD-ROM index.html files"
|
|
echo " "
|
|
echo " -d|--debug Enables (extremely verbose) debug output"
|
|
echo " "
|
|
echo " -f|--ftp Generates FTP index.html files"
|
|
echo " "
|
|
echo " -h|--help Displays this help message"
|
|
echo " "
|
|
echo " -q|--quiet Does not give progress output"
|
|
echo " "
|
|
echo " -p|--pkgsrc dir Specifies the pkgsrc directory. Defaults to"
|
|
echo " the value of the PKGSRCDIR environment variable"
|
|
echo " if set or /usr/pkgsrc otherwise."
|
|
echo " "
|
|
echo " -P|--packages dir Specifies the packages directory."
|
|
echo " "
|
|
echo " -r|--restart Restart. This option assumes that the database file"
|
|
echo " from a previous run still exists and that the script"
|
|
echo " should use that instead of recreating the database."
|
|
echo " "
|
|
echo " -s|--summary Generate pkg_summary.gz files for the binary packages"
|
|
echo " directories while processing them."
|
|
echo " "
|
|
echo " -S|--save-database Does not delete the database file after the run."
|
|
echo " This is useful for debugging or re-running this script"
|
|
echo " with the -r option."
|
|
echo " "
|
|
echo " -v|--version Displays the version of this script and exits."
|
|
echo " "
|
|
echo "Example: $prog -p /pub/pkgsrc/current/pkgsrc -P /pub/pkgsrc -f"
|
|
echo " "
|
|
}
|
|
|
|
clean_and_exit(){
|
|
if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" -a "x$save" = "xno" ]; then
|
|
rm -fr $TMPDIR
|
|
else
|
|
echo "Debugging output left in $TMP"
|
|
fi
|
|
exit 1
|
|
}
|
|
|
|
prog=$0
|
|
|
|
|
|
######################################################################
|
|
#
|
|
# Handle command line options
|
|
#
|
|
######################################################################
|
|
|
|
cmdline=$*
|
|
|
|
ftp_readme=no
|
|
restart=no
|
|
prune=no
|
|
DEBUG=no
|
|
save=no
|
|
pv=default
|
|
quiet=no
|
|
|
|
summary=""
|
|
|
|
while
|
|
test -n "$1"
|
|
do
|
|
case "$1"
|
|
in
|
|
|
|
# We're generating index.html's for a CD-ROM
|
|
-c|--cdrom)
|
|
ftp_readme=no
|
|
shift
|
|
;;
|
|
|
|
# Prune old index.html files from pkgs which no longer exist
|
|
-C|--prune)
|
|
prune=yes
|
|
shift
|
|
;;
|
|
|
|
# Turn on debugging
|
|
-d|--debug)
|
|
DEBUG=yes
|
|
shift
|
|
;;
|
|
|
|
# We're generating index.html's for a CD-ROM
|
|
-f|--ftp)
|
|
ftp_readme=yes
|
|
shift
|
|
;;
|
|
|
|
|
|
# Help
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
|
|
# be less talkative
|
|
-q|--quiet)
|
|
quiet=yes
|
|
shift
|
|
;;
|
|
|
|
# Specify pkgsrc directory
|
|
-p|--pkgsrc)
|
|
PKGSRCDIR=$2
|
|
shift 2
|
|
;;
|
|
|
|
# Specify PACKAGES directory
|
|
-P|--packages)
|
|
PKGDIR=$2
|
|
shift 2
|
|
;;
|
|
|
|
# Restart (ie, don't re-generate the database file)
|
|
-r|--restart)
|
|
restart=yes
|
|
shift
|
|
;;
|
|
|
|
# Generate the pkg_summary.gz files
|
|
-s|--summary)
|
|
summary="--summary"
|
|
shift
|
|
;;
|
|
|
|
# Save the database files
|
|
-S|--save-database)
|
|
save=yes
|
|
shift
|
|
;;
|
|
|
|
# Version
|
|
-v|--version)
|
|
${AWK} '/^#[ \t]*\$NetBSD/ {gsub(/,v/,"",$3);printf("%s: Version %s, %s\n",$3,$4,$5); exit 0;}' $prog
|
|
exit 0
|
|
;;
|
|
|
|
-*) echo "$prog: ERROR: $1 is not a valid option"
|
|
usage
|
|
clean_and_exit
|
|
;;
|
|
|
|
*)
|
|
break
|
|
;;
|
|
|
|
esac
|
|
done
|
|
|
|
if [ "x$DEBUG" = "xyes" ]; then
|
|
set -v
|
|
fi
|
|
|
|
if [ ! -d ${PKGSRCDIR} ]; then
|
|
echo "ERROR: package source directory ${PKGSRCDIR} does not exist"
|
|
echo ""
|
|
clean_and_exit
|
|
fi
|
|
|
|
if [ ! -d $TMPDIR ]; then
|
|
mkdir -p $TMPDIR
|
|
fi
|
|
|
|
DEPENDSTREEFILE=$TMPDIR/dependstree
|
|
export DEPENDSTREEFILE
|
|
DEPENDSFILE=$TMPDIR/depends
|
|
export DEPENDSFILE
|
|
SUPPORTSFILE=$TMPDIR/supports
|
|
export SUPPORTSFILE
|
|
INDEXFILE=$TMPDIR/index
|
|
export SUPPORTSFILE
|
|
ORDERFILE=$TMPDIR/order
|
|
export ORDERFILE
|
|
DATABASEFILE=$TMPDIR/database
|
|
export DATABASEFILE
|
|
BINPKGFILE=$TMPDIR/binpkglist
|
|
|
|
echo "Starting index.html generation: `date`"
|
|
|
|
######################################################################
|
|
#
|
|
# Extract key pkgsrc configuration variables
|
|
#
|
|
######################################################################
|
|
|
|
echo " "
|
|
echo "Extracting tool variables"
|
|
if [ -d ${PKGSRCDIR}/pkgtools/prereq-readme ]; then
|
|
cd ${PKGSRCDIR}/pkgtools/prereq-readme
|
|
eval "`${BMAKE} show-tools`"
|
|
else
|
|
echo "Error: ${PKGSRCDIR}/pkgtools/prereq-readme does not seem to exist"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Extracting configuration variables"
|
|
if [ -d ${PKGSRCDIR}/pkgtools/prereq-readme ]; then
|
|
cd ${PKGSRCDIR}/pkgtools/prereq-readme
|
|
for v in CDROM_PKG_URL_HOST CDROM_PKG_URL_DIR DISTDIR \
|
|
FTP_PKG_URL_HOST FTP_PKG_URL_DIR PACKAGES PKG_INFO PKG_SUFX PKG_ADMIN \
|
|
AUDIT_PACKAGES AUDIT_PACKAGES_FLAGS PKGTOOLS_VERSION
|
|
do
|
|
val=`${BMAKE} show-var VARNAME=${v}`
|
|
if [ $? != 0 ]; then
|
|
echo "Error: make show-var VARNAME=${v} in `pwd` "
|
|
echo "Failed. This is a fatal error"
|
|
clean_and_exit
|
|
fi
|
|
eval "${v}=\"${val}\""
|
|
done
|
|
else
|
|
echo "Error: ${PKGSRCDIR}/pkgtools/prereq-readme does not seem to exist"
|
|
exit 1
|
|
fi
|
|
|
|
if [ `${PKG_ADMIN} -V` -lt 20080415 ]; then
|
|
SCAN_VULNERABILITIES=0
|
|
echo "----> NOT checking for vulnerabilities, pkg_install too old"
|
|
else
|
|
_PVDIR=`${PKG_ADMIN} config-var PKGVULNDIR`;
|
|
if [ -e "${_PVDIR}"/pkg-vulnerabilities ]; then
|
|
SCAN_VULNERABILITIES=2
|
|
echo "----> Checking for vulnerabilities"
|
|
else
|
|
SCAN_VULNERABILITIES=1
|
|
echo "----> NOT checking for vulnerabilities"
|
|
fi
|
|
fi
|
|
|
|
######################################################################
|
|
#
|
|
# Decide on FTP vs CDROM index.html files
|
|
#
|
|
######################################################################
|
|
|
|
if [ "$ftp_readme" = "yes" ]; then
|
|
PKG_URL=${FTP_PKG_URL_HOST}${FTP_PKG_URL_DIR}
|
|
echo "Will generate FTP readme files with PKG_URL=$PKG_URL"
|
|
else
|
|
PKG_URL=${CDROM_PKG_URL_HOST}${CDROM_PKG_URL_DIR}
|
|
echo "Will generate CD-ROM readme files with PKG_URL=$PKG_URL"
|
|
fi
|
|
|
|
######################################################################
|
|
#
|
|
# Check for command line switch for packages directory
|
|
#
|
|
######################################################################
|
|
|
|
# we've been given the directory as a command line switch
|
|
if [ ! -z "$PKGDIR" ]; then
|
|
PACKAGES=$PKGDIR
|
|
echo "PACKAGES specified on command line to be $PKGDIR"
|
|
fi
|
|
|
|
|
|
######################################################################
|
|
#
|
|
# Extract Database for All Packages (longest step)
|
|
#
|
|
######################################################################
|
|
|
|
if [ "x$restart" = "xno" ] ; then
|
|
echo " "
|
|
echo "Extracting data. This could take a while"
|
|
echo "Started at: `date` "
|
|
echo " "
|
|
npkg=1
|
|
|
|
# make sure we don't have an old database lying around
|
|
rm -fr $DATABASEFILE
|
|
|
|
cd ${PKGSRCDIR}
|
|
cats=`${BMAKE} show-subdir-var VARNAME=SUBDIR`
|
|
for c in ${cats} ; do
|
|
if [ ! -d ${PKGSRCDIR}/${c} ]; then
|
|
echo "WARNING: The category directory ${c} does not seem to" > /dev/stderr
|
|
echo " exist under ${PKGSRCDIR}" > /dev/stderr
|
|
else
|
|
if [ "x$quiet" = "xno" ]; then
|
|
echo " "
|
|
echo "Extracting data for category ${c}"
|
|
fi
|
|
cd ${PKGSRCDIR}/${c}
|
|
list=`${BMAKE} show-subdir-var VARNAME=SUBDIR`
|
|
for pkgdir in $list ; do
|
|
cd ${PKGSRCDIR}/${c}
|
|
if [ ! -d $pkgdir ]; then
|
|
echo " "
|
|
echo "WARNING: the package directory $pkgdir is listed in" > /dev/stderr
|
|
echo " ${PKGSRCDIR}/${c}/Makefile" > /dev/stderr
|
|
echo " but the directory does not exist. Please fix this!" > /dev/stderr
|
|
else
|
|
cd ${PKGSRCDIR}/${c}/${pkgdir}
|
|
o=`${BMAKE} show-options | ${AWK} -f ${PKGSRCDIR}/mk/scripts/htmloptions.awk`
|
|
l=`${BMAKE} print-summary-data`
|
|
if [ $? != 0 ]; then
|
|
echo "WARNING (printdepends): the package in ${c}/${pkgdir} had problem with" \
|
|
> /dev/stderr
|
|
echo " ${BMAKE} print-summary-data" > /dev/stderr
|
|
echo " database information for this package" > /dev/stderr
|
|
echo " will be dropped." > /dev/stderr
|
|
${BMAKE} print-summary-data 2>&1 > /dev/stderr
|
|
else
|
|
echo "$l" >> $DATABASEFILE
|
|
echo "htmloptions ${c}/${pkgdir} $o" >> $DATABASEFILE
|
|
fi
|
|
fi
|
|
if [ "x$quiet" = "xno" ]; then
|
|
${ECHO} -n "."
|
|
if [ `${EXPR} $npkg % 100 = 0` -eq 1 ]; then
|
|
echo " "
|
|
echo "$npkg"
|
|
fi
|
|
fi
|
|
npkg=`${EXPR} $npkg + 1`
|
|
cd ${PKGSRCDIR}
|
|
done
|
|
fi
|
|
done
|
|
echo " "
|
|
echo "Finished extracting data for ${npkg} packages at: `date` "
|
|
else
|
|
echo " "
|
|
echo "Using existing database (are you sure you wanted the -r/--restart flag?)"
|
|
echo " "
|
|
if [ ! -f $DATABASEFILE ]; then
|
|
echo " "
|
|
echo "ERROR: You have use the -r/--restart flag but the database "
|
|
echo " file $DATABASEFILE does not exist"
|
|
echo " "
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
######################################################################
|
|
#
|
|
# Generate the package and category index.html files
|
|
#
|
|
######################################################################
|
|
|
|
echo " "
|
|
echo "Generating package indexx.html files"
|
|
echo " "
|
|
if [ "x$DEBUG" = "xyes" ]; then
|
|
debug=1;
|
|
else
|
|
debug=0;
|
|
fi
|
|
|
|
${AWK} -f ${PKGSRCDIR}/mk/scripts/genreadme.awk \
|
|
builddependsfile=${TMPDIR}/pkgsrc.builddepends.debug \
|
|
debug=$debug \
|
|
quiet=$quiet \
|
|
dependsfile=${TMPDIR}/pkgsrc.depends.debug \
|
|
summary=${summary} \
|
|
AWK=$AWK \
|
|
CMP=$CMP \
|
|
DISTDIR=$DISTDIR \
|
|
FIND=$FIND \
|
|
GREP=$GREP \
|
|
GZIP_CMD="$GZIP_CMD" \
|
|
PACKAGES=$PACKAGES \
|
|
PKG_ADMIN="$PKG_ADMIN" \
|
|
PKG_INFO="$PKG_INFO" \
|
|
PKG_SUFX=$PKG_SUFX \
|
|
PKG_URL=$PKG_URL \
|
|
PKGSRCDIR=$PKGSRCDIR \
|
|
PKGTOOLS_VERSION=$PKGTOOLS_VERSION \
|
|
SCAN_VULNERABILITIES=${SCAN_VULNERABILITIES} \
|
|
SED=$SED \
|
|
SETENV=$SETENV \
|
|
SORT=$SORT \
|
|
TMPDIR=$TMPDIR \
|
|
${DATABASEFILE}
|
|
|
|
if [ $? != 0 ]; then
|
|
echo "Error: genreadme.awk failed to create index.html files"
|
|
clean_and_exit
|
|
fi
|
|
|
|
######################################################################
|
|
#
|
|
# Prune index.html files which are no longer needed
|
|
#
|
|
######################################################################
|
|
if [ "x$prune" = "xyes" ]; then
|
|
echo " "
|
|
echo "Pruning unused index.html files"
|
|
echo " "
|
|
cd ${PKGSRCDIR}
|
|
# cats=`${BMAKE} show-subdir-var VARNAME=SUBDIR` # already set above
|
|
for c in ${cats} ; do
|
|
for d in `ls -d ${c}/*` ; do
|
|
if [ -d $d -a ! -f ${d}/Makefile -a -f ${d}/index.html ]; then
|
|
echo "Pruning ${d}/index.html which is no longer used"
|
|
rm -f ${d}/index.html
|
|
fi
|
|
done
|
|
done
|
|
fi
|
|
|
|
######################################################################
|
|
#
|
|
# All done. Clean (if needed) and exit
|
|
#
|
|
######################################################################
|
|
echo " "
|
|
echo "index.html generation finished: `date`"
|
|
echo " "
|
|
if [ "x$DEBUG" = "xno" -a "x$restart" = "xno" -a "x$save" = "xno" ]; then
|
|
rm -fr $TMPDIR
|
|
else
|
|
echo "Debugging output left in $TMPDIR"
|
|
fi
|
|
|