004b23284b
as an acceptable input. Fixes Pkgsrc for users that set PKG_RESUME_TRANSFERS=yes in their mk.conf if installing a package that has an interactive fetch stage, after mk/fetch/fetch.mk revision 1.66. Tested for all combinations of: + PKG_RESUME_TRANSFERS=yes/no; and + interactive fetch=yes/no; and + distfile=downloaded/not downloaded ... with no signs of misbehaviour. The specific case that was broken, and this change fixes, is: + PKG_RESUME_TRANSFERS=yes; and + interactive fetch=yes; and + distfile=downloaded This change was designed with the following considerations: 1. Given it's a freeze, keep it simple. 2. The change in behaviour allows 'fetch' to succeed where it would previously fail fatally, so it's unlikely to affect any (intentional) existing behaviour in Pkgsrc. 3. The behaviour of 'fetch' with zero sites is essentially the same as a fetch where all the possible download sites fail, I.e. Pkgsrc already expects to handle such behaviour. ok gdt@
318 lines
9.1 KiB
Bash
Executable file
318 lines
9.1 KiB
Bash
Executable file
#!/bin/sh
|
|
#
|
|
# $NetBSD: fetch,v 1.18 2015/09/19 11:45:56 dsainty Exp $
|
|
#
|
|
# Copyright (c) 2006, 2015 The NetBSD Foundation, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This code is derived from software contributed to The NetBSD Foundation
|
|
# by Johnny C. Lam.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions
|
|
# are met:
|
|
# 1. Redistributions of source code must retain the above copyright
|
|
# notice, this list of conditions and the following disclaimer.
|
|
# 2. Redistributions in binary form must reproduce the above copyright
|
|
# notice, this list of conditions and the following disclaimer in the
|
|
# documentation and/or other materials provided with the distribution.
|
|
# 3. All advertising materials mentioning features or use of this software
|
|
# must display the following acknowledgement:
|
|
# This product includes software developed by the NetBSD
|
|
# Foundation, Inc. and its contributors.
|
|
# 4. Neither the name of The NetBSD Foundation nor the names of its
|
|
# contributors may be used to endorse or promote products derived
|
|
# from this software without specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
|
|
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
|
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
|
|
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
|
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
|
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
|
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
|
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
# POSSIBILITY OF SUCH DAMAGE.
|
|
#
|
|
|
|
######################################################################
|
|
#
|
|
# NAME
|
|
# fetch -- fetch files via URLs
|
|
#
|
|
# SYNOPSIS
|
|
# fetch [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file [site ...]
|
|
#
|
|
# DESCRIPTION
|
|
# fetch will attempt to fetch the file from the list of specified
|
|
# sites in the order given. The complete URL to the file on each
|
|
# site should be the concatenation of the specified site and file.
|
|
# If the file cannot be fetched successfully, then we try the next
|
|
# listed site.
|
|
#
|
|
# If the file already exists on the disk and is verified, then
|
|
# no fetch action is taken.
|
|
#
|
|
# OPTIONS
|
|
# -c Verify the checksum for the file. If the checksum
|
|
# does not match, then the fetch is determined to be
|
|
# not successful.
|
|
#
|
|
# -d dir Fetch the files into the specified directory.
|
|
#
|
|
# -f distinfo
|
|
# The path to the distinfo file containing the checksums
|
|
# for the file. The file format should match what is
|
|
# needed by the pkgsrc/mk/checksum/checksum script.
|
|
#
|
|
# -p hook
|
|
# After a successful fetch, run hook. The first argument is
|
|
# the relative path of the distfile and the second argument
|
|
# the full URL the file was obtained from.
|
|
#
|
|
# -r Resume a previous fetch for the file. In this case,
|
|
# the file is first saved to a ".pkgsrc.resume" file,
|
|
# and is later renamed to the final file name if the
|
|
# complete file has been sucessfully fetched.
|
|
#
|
|
# -v Show the actual command line used to fetch the file
|
|
# from each site.
|
|
#
|
|
# ENVIRONMENT
|
|
# PKGSRCDIR This is a hint to help locate the default
|
|
# checksum script.
|
|
#
|
|
# CHECKSUM This is the path to the checksum script used
|
|
# when "-c" is specified.
|
|
#
|
|
# FETCH_CMD This is the actual command used for transferring
|
|
# files from the various sites.
|
|
#
|
|
# The following are lists of options to pass to ${FETCH_CMD}:
|
|
#
|
|
# FETCH_BEFORE_ARGS
|
|
# These options appear before all other options.
|
|
#
|
|
# FETCH_AFTER_ARGS
|
|
# These options appear after all other options.
|
|
#
|
|
# FETCH_RESUME_ARGS
|
|
# These options appear just after FETCH_BEFORE_ARGS
|
|
# options and cause ${FETCH_CMD} to resume a
|
|
# previous file transfer.
|
|
#
|
|
# FETCH_OUTPUT_ARGS
|
|
# These options specify the name of the local file
|
|
# that will hold the contents of the fetched file.
|
|
#
|
|
######################################################################
|
|
|
|
: ${PKGSRCDIR:=/usr/pkgsrc}
|
|
: ${CHECKSUM:=${PKGSRCDIR}/mk/checksum/checksum}
|
|
: ${CP:=cp}
|
|
: ${ECHO:=echo}
|
|
: ${FETCH_CMD:=ftp}
|
|
: ${MKDIR:=mkdir}
|
|
: ${MV:=mv}
|
|
: ${TEST:=test}
|
|
: ${TOUCH:=touch}
|
|
: ${WC:=wc}
|
|
|
|
self="${0##*/}"
|
|
|
|
usage() {
|
|
${ECHO} 1>&2 "usage: $self [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file [site ...]"
|
|
}
|
|
|
|
# Process optional arguments
|
|
checksum=
|
|
distinfo=
|
|
fetchdir=. # A relative directory or "."
|
|
post_fetch=
|
|
resume=
|
|
verbose=
|
|
while ${TEST} $# -gt 0; do
|
|
case "$1" in
|
|
-c) checksum=yes; shift ;;
|
|
-d) fetchdir="$2"; shift 2 ;;
|
|
-f) distinfo="$2"; shift 2 ;;
|
|
-p) post_fetch="$2"; shift 2 ;;
|
|
-r) resume=yes; shift ;;
|
|
-v) verbose=yes; shift ;;
|
|
--) shift; break ;;
|
|
-*) ${ECHO} 1>&2 "$self: unknown option -- ${1#-}"
|
|
usage
|
|
exit 1
|
|
;;
|
|
*) break ;;
|
|
esac
|
|
done
|
|
if ${TEST} -n "$checksum" -a -z "$distinfo"; then
|
|
${ECHO} 1>&2 "$self: \`\`-c'' requires \`\`-f distinfo''."
|
|
exit 1
|
|
fi
|
|
if ${TEST} -n "$resume"; then
|
|
if ${TEST} -z "$distinfo"; then
|
|
${ECHO} 1>&2 "$self: \`\`-r'' requires \`\`-f distinfo''."
|
|
resume=
|
|
elif ${TEST} "x${FETCH_RESUME_ARGS}" = "x"; then
|
|
${ECHO} 1>&2 "$self: \`\`-r'' requires FETCH_RESUME_ARGS to be non-empty."
|
|
resume=
|
|
fi
|
|
${TEST} -n "$resume" ||
|
|
${ECHO} 1>&2 "$self: Falling back to non-resume fetch."
|
|
fi
|
|
|
|
# Process required arguments
|
|
if ${TEST} $# -lt 1; then
|
|
usage
|
|
exit 1
|
|
fi
|
|
file="$1"; shift
|
|
path="$fetchdir/$file"
|
|
|
|
if ${TEST} -n "$distinfo" && ${TEST} ! -f "$distinfo"; then
|
|
${ECHO} 1>&2 "$self: distinfo file missing: $distinfo"
|
|
exit 1
|
|
fi
|
|
|
|
# Compute the expected size of the fetched file.
|
|
distsize=
|
|
distunits=
|
|
if ${TEST} -n "$distinfo"; then
|
|
while read d_type d_file d_equals d_size d_units; do
|
|
case "$d_type" in
|
|
Size) ;; # only handle "Size" lines
|
|
*) continue ;;
|
|
esac
|
|
case "$fetchdir" in
|
|
".") ${TEST} "$d_file" = "($file)" || continue ;;
|
|
*) ${TEST} "$d_file" = "($path)" || continue ;;
|
|
esac
|
|
distsize="$d_size"; distunits="$d_units"
|
|
break
|
|
done < $distinfo
|
|
fi
|
|
|
|
# verify_file [-v] $file
|
|
# If we can checksum the file, then see if it matches the listed
|
|
# checksums in the distinfo file. If we can check the size, then
|
|
# check that instead. We strip off ".pkgsrc.resume" from the
|
|
# filename so that we can verify the checksum for the temporary
|
|
# fetch file as well.
|
|
#
|
|
verify_file() {
|
|
_if_verbose=:; if [ "x$1" = "x-v" ]; then shift; _if_verbose=; fi
|
|
_file="${1#./}"
|
|
${TEST} -f $_file || {
|
|
$_if_verbose ${ECHO} 1>&2 "$self: File $_file does not exist."
|
|
return 1
|
|
}
|
|
if ${TEST} -n "$checksum"; then
|
|
${CHECKSUM} -s ".pkgsrc.resume" $distinfo ${_file} || {
|
|
$_if_verbose ${ECHO} 1>&2 "$self: Checksum of the file $_file doesn't match."
|
|
return 1
|
|
}
|
|
return 0
|
|
elif ${TEST} -n "$distsize"; then
|
|
_size=`${WC} -c < $_file`
|
|
${TEST} "$_size" -eq "$distsize" || {
|
|
$_if_verbose ${ECHO} 1>&2 "$self: Size of the file $_file doesn't match."
|
|
return 1
|
|
}
|
|
return 0
|
|
fi
|
|
return 0;
|
|
}
|
|
|
|
# If the file already exists and it verifies, then we don't need to fetch
|
|
# it again.
|
|
#
|
|
if verify_file $path; then
|
|
exit 0
|
|
fi
|
|
|
|
${TEST} -d $fetchdir || ${MKDIR} -p $fetchdir 2>/dev/null
|
|
if ${TEST} ! -w $fetchdir; then
|
|
${ECHO} 1>&2 "$self: Cannot write to `cd $fetchdir && pwd`"
|
|
exit 1
|
|
fi
|
|
|
|
# Set the name of the output file. In the "resume" case, we initialize
|
|
# the fetch loop by ensuring that the temporary output file already
|
|
# exists.
|
|
#
|
|
outputfile="$file"
|
|
outputpath="$fetchdir/$outputfile"
|
|
if ${TEST} -n "$resume"; then
|
|
outputfile="${file}.pkgsrc.resume"
|
|
outputpath="$fetchdir/$outputfile"
|
|
if ${TEST} ! -f $outputpath; then
|
|
if ${TEST} -f $path; then
|
|
${CP} -f $path $outputpath
|
|
else
|
|
${RM} -f $outputpath
|
|
${TOUCH} $outputpath
|
|
fi
|
|
fi
|
|
#
|
|
# If the temporary file verifies, then we don't need to resume
|
|
# fetching it.
|
|
#
|
|
if verify_file $outputpath; then
|
|
${MV} -f $outputpath $path
|
|
exit 0
|
|
fi
|
|
size=`${WC} -c < $outputpath`
|
|
${ECHO} "=> Downloaded size: $size bytes"
|
|
fi
|
|
${TEST} -z "$distsize" || ${ECHO} "=> Total size: $distsize $distunits"
|
|
|
|
# Iterate over each site and try to fetch the file. We verify the fetched
|
|
# file to see if we need to try fetching from the next site.
|
|
#
|
|
while ${TEST} $# -gt 0; do
|
|
site="$1"; shift
|
|
case "$site" in
|
|
-*)
|
|
url=${site#-}
|
|
;;
|
|
*)
|
|
url=$site$file
|
|
;;
|
|
esac
|
|
|
|
( cd $fetchdir
|
|
if ${TEST} -n "$resume"; then
|
|
fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $url"
|
|
else
|
|
fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $url ${FETCH_AFTER_ARGS}"
|
|
fi
|
|
${TEST} -z "$verbose" || ${ECHO} "$fetch_cmd"
|
|
$fetch_cmd )
|
|
if ${TEST} $? -ne 0; then
|
|
${ECHO} 1>&2 "$self: Unable to fetch expected file $file"
|
|
continue
|
|
fi
|
|
if verify_file -v $outputpath; then
|
|
${TEST} -z "$resume" || ${MV} -f $outputpath $path
|
|
break
|
|
fi
|
|
if ${TEST} -n "$resume"; then
|
|
if ${TEST} -f $path; then
|
|
${CP} -f $path $outputpath
|
|
else
|
|
${TOUCH} $outputpath
|
|
fi
|
|
fi
|
|
done
|
|
if ${TEST} -f $path && ${TEST} -n "$post_fetch"; then
|
|
"$post_fetch" $path "$url"
|
|
fi
|
|
if ${TEST} -f $path; then
|
|
exit 0
|
|
else
|
|
exit 1
|
|
fi
|