kpriv/pkgsrc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity
pkgsrc/mk/fetch/fetch
schmonz 318812e1f2 The non-writability check for DISTDIR is intended to prevent wasting 
resources when fetchers wouldn't be able to save what they download. On
my build farm with pkgsrc (and pkgsrc/distfiles) mounted over NFS,
however, the check gives false positives for NetBSD 9.2 and -current.
Downgrade it to a warning so that these fetches can succeed, while
leaving a breadcrumb in case someone encounters a true positive.

It would of course be interesting to sort out why, in my environment, a
wide variety of other OSes get 1 for "${TEST} ! -w $fetchdir" while
NetBSD gets 0. In the meantime, joerg@ suggested this workaround and
gdt@ agrees it's reasonable to try.
2021-12-16 14:10:46 +00:00

309 lines
8.6 KiB
Bash
Executable file
Raw Blame History

#!/bin/sh
#
# $NetBSD: fetch,v 1.21 2021/12/16 14:10:46 schmonz Exp $
#
# Copyright (c) 2006, 2015 The NetBSD Foundation, Inc.
# All rights reserved.
#
# This code is derived from software contributed to The NetBSD Foundation
# by Johnny C. Lam.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
######################################################################
#
# NAME
# fetch -- fetch files via URLs
#
# SYNOPSIS
# fetch [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file [site ...]
#
# DESCRIPTION
# fetch will attempt to fetch the file from the list of specified
# sites in the order given. The complete URL to the file on each
# site should be the concatenation of the specified site and file.
# If the file cannot be fetched successfully, then we try the next
# listed site.
#
# If the file already exists on the disk and is verified, then
# no fetch action is taken.
#
# OPTIONS
# -c Verify the checksum for the file. If the checksum
# does not match, then the fetch is determined to be
# not successful.
#
# -d dir Fetch the files into the specified directory.
#
# -f distinfo
# The path to the distinfo file containing the checksums
# for the file. The file format should match what is
# needed by the pkgsrc/mk/checksum/checksum script.
#
# -p hook
# After a successful fetch, run hook. The first argument is
# the relative path of the distfile and the second argument
# the full URL the file was obtained from.
#
# -r Resume a previous fetch for the file. In this case,
# the file is first saved to a ".pkgsrc.resume" file,
# and is later renamed to the final file name if the
# complete file has been sucessfully fetched.
#
# -v Show the actual command line used to fetch the file
# from each site.
#
# ENVIRONMENT
# PKGSRCDIR This is a hint to help locate the default
# checksum script.
#
# CHECKSUM This is the path to the checksum script used
# when "-c" is specified.
#
# FETCH_CMD This is the actual command used for transferring
# files from the various sites.
#
# The following are lists of options to pass to ${FETCH_CMD}:
#
# FETCH_BEFORE_ARGS
# These options appear before all other options.
#
# FETCH_AFTER_ARGS
# These options appear after all other options.
#
# FETCH_RESUME_ARGS
# These options appear just after FETCH_BEFORE_ARGS
# options and cause ${FETCH_CMD} to resume a
# previous file transfer.
#
# FETCH_OUTPUT_ARGS
# These options specify the name of the local file
# that will hold the contents of the fetched file.
#
######################################################################
: ${PKGSRCDIR:=/usr/pkgsrc}
: ${CHECKSUM:=false}
: ${CP:=cp}
: ${ECHO:=echo}
: ${FETCH_CMD:=ftp}
: ${MKDIR:=mkdir}
: ${MV:=mv}
: ${TEST:=test}
: ${TOUCH:=touch}
: ${WC:=wc}
self="${0##*/}"
usage() {
${ECHO} 1>&2 "usage: $self [-c] [-d dir] [-f distinfo] [-p hook] [-r] [-v] file [site ...]"
}
# Process optional arguments
checksum=
distinfo=
fetchdir=. # A relative directory or "."
post_fetch=
resume=
verbose=
while ${TEST} $# -gt 0; do
case "$1" in
-c) checksum=yes; shift ;;
-d) fetchdir="$2"; shift 2 ;;
-f) distinfo="$2"; shift 2 ;;
-p) post_fetch="$2"; shift 2 ;;
-r) resume=yes; shift ;;
-v) verbose=yes; shift ;;
--) shift; break ;;
-*) ${ECHO} 1>&2 "$self: unknown option -- ${1#-}"
usage
exit 1
;;
*) break ;;
esac
done
if ${TEST} -n "$checksum" -a -z "$distinfo"; then
${ECHO} 1>&2 "$self: \`\`-c'' requires \`\`-f distinfo''."
exit 1
fi
if ${TEST} -n "$resume"; then
if ${TEST} -z "$distinfo"; then
${ECHO} 1>&2 "$self: \`\`-r'' requires \`\`-f distinfo''."
resume=
elif ${TEST} "x${FETCH_RESUME_ARGS}" = "x"; then
${ECHO} 1>&2 "$self: \`\`-r'' requires FETCH_RESUME_ARGS to be non-empty."
resume=
fi
${TEST} -n "$resume" ||
${ECHO} 1>&2 "$self: Falling back to non-resume fetch."
fi
# Process required arguments
if ${TEST} $# -lt 1; then
usage
exit 1
fi
file="$1"; shift
path="$fetchdir/$file"
if ${TEST} -n "$distinfo" && ${TEST} ! -f "$distinfo"; then
${ECHO} 1>&2 "$self: distinfo file missing: $distinfo"
exit 1
fi
# Compute the expected size of the fetched file.
distsize=
distunits=
if ${TEST} -n "$distinfo"; then
while read d_type d_file d_equals d_size d_units; do
case "$d_type" in
Size) ;; # only handle "Size" lines
*) continue ;;
esac
case "$fetchdir" in
".") ${TEST} "$d_file" = "($file)" || continue ;;
*) ${TEST} "$d_file" = "($path)" || continue ;;
esac
distsize="$d_size"; distunits="$d_units"
break
done < $distinfo
fi
# verify_file [-v] $file
# If we can checksum the file, then see if it matches the listed
# checksums in the distinfo file. If we can check the size, then
# check that instead. We strip off ".pkgsrc.resume" from the
# filename so that we can verify the checksum for the temporary
# fetch file as well.
#
verify_file() {
_if_verbose=:; if [ "x$1" = "x-v" ]; then shift; _if_verbose=; fi
_file="${1#./}"
${TEST} -f $_file || {
$_if_verbose ${ECHO} 1>&2 "$self: File $_file does not exist."
return 1
}
if ${TEST} -n "$checksum"; then
${CHECKSUM} -s ".pkgsrc.resume" $distinfo ${_file} || {
$_if_verbose ${ECHO} 1>&2 "$self: Checksum of the file $_file doesn't match."
return 1
}
return 0
elif ${TEST} -n "$distsize"; then
_size=`${WC} -c < $_file`
${TEST} "$_size" -eq "$distsize" || {
$_if_verbose ${ECHO} 1>&2 "$self: Size of the file $_file doesn't match."
return 1
}
return 0
fi
return 0;
}
# If the file already exists and it verifies, then we don't need to fetch
# it again.
#
if verify_file $path; then
exit 0
fi
${TEST} -d $fetchdir || ${MKDIR} -p $fetchdir 2>/dev/null
${TEST} -w $fetchdir || ${ECHO} 1>&2 "$self: WARNING: DISTDIR `cd $fetchdir && pwd` looks non-writable."
# Set the name of the output file. In the "resume" case, we initialize
# the fetch loop by ensuring that the temporary output file already
# exists.
#
outputfile="$file"
outputpath="$fetchdir/$outputfile"
if ${TEST} -n "$resume"; then
outputfile="${file}.pkgsrc.resume"
outputpath="$fetchdir/$outputfile"
if ${TEST} ! -f $outputpath; then
if ${TEST} -f $path; then
${CP} -f $path $outputpath
else
${RM} -f $outputpath
${TOUCH} $outputpath
fi
fi
#
# If the temporary file verifies, then we don't need to resume
# fetching it.
#
if verify_file $outputpath; then
${MV} -f $outputpath $path
exit 0
fi
size=`${WC} -c < $outputpath`
${ECHO} "=> Downloaded size: $size bytes"
fi
${TEST} -z "$distsize" || ${ECHO} "=> Total size: $distsize $distunits"
# Iterate over each site and try to fetch the file. We verify the fetched
# file to see if we need to try fetching from the next site.
#
while ${TEST} $# -gt 0; do
site="$1"; shift
case "$site" in
-*)
url=${site#-}
;;
*)
url=$site$file
;;
esac
( cd $fetchdir
if ${TEST} -n "$resume"; then
fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_RESUME_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $url"
else
fetch_cmd="${FETCH_CMD} ${FETCH_BEFORE_ARGS} ${FETCH_OUTPUT_ARGS} $outputfile $url ${FETCH_AFTER_ARGS}"
fi
${TEST} -z "$verbose" || ${ECHO} "$fetch_cmd"
$fetch_cmd )
if ${TEST} $? -ne 0; then
${ECHO} 1>&2 "$self: Unable to fetch expected file $file"
continue
fi
if verify_file -v $outputpath; then
${TEST} -z "$resume" || ${MV} -f $outputpath $path
break
fi
if ${TEST} -n "$resume"; then
if ${TEST} -f $path; then
${CP} -f $path $outputpath
else
${TOUCH} $outputpath
fi
fi
done
if ${TEST} -f $path && ${TEST} -n "$post_fetch"; then
"$post_fetch" $path "$url"
fi
if ${TEST} -f $path; then
exit 0
else
exit 1
fi
Reference in a new issue View git blame Copy permalink