0f000d2afb
lead to overloads of very first distribution site. Moreover, if first site in the list is not available (often seen for sourceforge mirrors) you have to wait for timeout each time. To distribute load on master distribution sites and to make second problem not so annoying randomly intermix list of MASTER_SITES with MASTER_SORT_RANDOM feature. Any of MASTER_SORT and MASTER_SORT_REGEX can be applied later. The feature is turned ON by default and is disabled for PKG_DEVELOPERs or if MASTER_SORT_RANDOM=no.
307 lines
11 KiB
Makefile
307 lines
11 KiB
Makefile
# $NetBSD: fetch.mk,v 1.22 2006/10/08 11:37:38 mishka Exp $
|
|
|
|
_MASTER_SITE_BACKUP= ${MASTER_SITE_BACKUP:=${DIST_SUBDIR}${DIST_SUBDIR:D/}}
|
|
_MASTER_SITE_OVERRIDE= ${MASTER_SITE_OVERRIDE:=${DIST_SUBDIR}${DIST_SUBDIR:D/}}
|
|
|
|
# Where to put distfiles that don't have any other master site
|
|
MASTER_SITE_LOCAL?= ${MASTER_SITE_BACKUP:=LOCAL_PORTS/}
|
|
|
|
ALLFILES?= ${DISTFILES} ${PATCHFILES}
|
|
ALLFILES:= ${ALLFILES:O:u} # remove duplicates
|
|
CKSUMFILES?= ${ALLFILES}
|
|
.for __tmp__ in ${IGNOREFILES}
|
|
CKSUMFILES:= ${CKSUMFILES:N${__tmp__}}
|
|
.endfor
|
|
|
|
# List of all files, with ${DIST_SUBDIR} in front. Used for fetch and checksum.
|
|
.if defined(DIST_SUBDIR)
|
|
_CKSUMFILES?= ${CKSUMFILES:@.f.@${DIST_SUBDIR}/${.f.}@}
|
|
_DISTFILES?= ${DISTFILES:@.f.@${DIST_SUBDIR}/${.f.}@}
|
|
_IGNOREFILES?= ${IGNOREFILES:@.f.@${DIST_SUBDIR}/${.f.}@}
|
|
_PATCHFILES?= ${PATCHFILES:@.f.@${DIST_SUBDIR}/${.f.}@}
|
|
.else
|
|
_CKSUMFILES?= ${CKSUMFILES}
|
|
_DISTFILES?= ${DISTFILES}
|
|
_IGNOREFILES?= ${IGNOREFILES}
|
|
_PATCHFILES?= ${PATCHFILES}
|
|
.endif
|
|
_ALLFILES?= ${_DISTFILES} ${_PATCHFILES}
|
|
|
|
_BUILD_DEFS+= _DISTFILES _PATCHFILES
|
|
|
|
# Set up _ORDERED_SITES to work out the exact list of sites for every file,
|
|
# using the dynamic sites script, or ordering according to the master site
|
|
# list, MASTER_SORT_RANDOM randomization feature, or the patterns in
|
|
# MASTER_SORT or MASTER_SORT_REGEX as appropriate.
|
|
# No actual sorting is done until _ORDERED_SITES is expanded.
|
|
#
|
|
.if defined(MASTER_SORT_RANDOM) && !empty(MASTER_SORT_RANDOM:M[yY][eE][sS])
|
|
_MASTER_RAND_AWK= BEGIN { srand(seed) } { \
|
|
n = split($$0, site); \
|
|
for (i = n; i > 0; i--) { \
|
|
ir = int(rand() * i + 1); \
|
|
t = site[i]; site[i] = site[ir]; site[ir] = t; \
|
|
print site[i]; } }
|
|
_RAND_SITES_CMD= | ${AWK} -v seed=$$$$ '${_MASTER_RAND_AWK}'
|
|
.endif
|
|
|
|
.if defined(MASTER_SORT) || defined(MASTER_SORT_REGEX)
|
|
MASTER_SORT?=
|
|
MASTER_SORT_REGEX?=
|
|
MASTER_SORT_REGEX+= ${MASTER_SORT:S/./\\./g:C/.*/:\/\/[^\/]*&\//}
|
|
|
|
_MASTER_SORT_AWK= BEGIN { RS = " "; ORS = " "; IGNORECASE = 1 ; gl = "${MASTER_SORT_REGEX:S/\\/\\\\/g}"; }
|
|
. for srt in ${MASTER_SORT_REGEX}
|
|
_MASTER_SORT_AWK+= /${srt:C/\//\\\//g}/ { good["${srt:S/\\/\\\\/g}"] = good["${srt:S/\\/\\\\/g}"] " " $$0 ; next; }
|
|
. endfor
|
|
_MASTER_SORT_AWK+= { rest = rest " " $$0; } END { n=split(gl, gla); for(i=1;i<=n;i++) { print good[gla[i]]; } print rest; }
|
|
_SORT_SITES_CMD+= | ${AWK} '${_MASTER_SORT_AWK}'
|
|
.endif
|
|
|
|
.if defined(_RAND_SITES_CMD) || defined(_SORT_SITES_CMD)
|
|
_SORT_SITES_FULL_CMD= ${ECHO} $$unsorted_sites ${_RAND_SITES_CMD} ${_SORT_SITES_CMD}
|
|
_ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} `${_SORT_SITES_FULL_CMD:S/\\/\\\\/g:C/"/\"/g}`
|
|
.else
|
|
_ORDERED_SITES= ${_MASTER_SITE_OVERRIDE} $$unsorted_sites
|
|
.endif
|
|
|
|
#
|
|
# Associate each file to fetch with the correct site(s).
|
|
#
|
|
DYNAMIC_SITES_SCRIPT?= ${FILESDIR}/getsite.sh
|
|
DYNAMIC_SITES_CMD?= ${SETENV} PATH=${PATH:Q} ${SH} ${DYNAMIC_SITES_SCRIPT}
|
|
.if exists(${DYNAMIC_SITES_SCRIPT})
|
|
. for fetchfile in ${_ALLFILES}
|
|
SITES.${fetchfile:T:S/=/--/}?= `${DYNAMIC_SITES_CMD} ${fetchfile:T}`
|
|
. endfor
|
|
.endif
|
|
.if !empty(_DISTFILES)
|
|
. for fetchfile in ${_DISTFILES}
|
|
SITES.${fetchfile:T:S/=/--/}?= ${MASTER_SITES}
|
|
. endfor
|
|
.endif
|
|
.if !empty(_PATCHFILES)
|
|
. for fetchfile in ${_PATCHFILES}
|
|
SITES.${fetchfile:T:S/=/--/}?= ${PATCH_SITES}
|
|
. endfor
|
|
.endif
|
|
|
|
######################################################################
|
|
### fetch (PUBLIC)
|
|
######################################################################
|
|
### fetch is a public target to fetch all of the package distribution
|
|
### files.
|
|
###
|
|
_FETCH_TARGETS+= bootstrap-depends
|
|
_FETCH_TARGETS+= check-vulnerable
|
|
_FETCH_TARGETS+= pre-fetch
|
|
_FETCH_TARGETS+= do-fetch
|
|
_FETCH_TARGETS+= post-fetch
|
|
|
|
.PHONY: fetch
|
|
.if !target(fetch)
|
|
fetch: ${_FETCH_TARGETS}
|
|
.endif
|
|
|
|
######################################################################
|
|
### pre-fetch, do-fetch, post-fetch (PUBLIC, override)
|
|
######################################################################
|
|
### {pre,do,post}-fetch are the heart of the package-customizable
|
|
### fetch targets, and may be overridden within a package Makefile.
|
|
###
|
|
.PHONY: pre-fetch do-fetch post-fetch
|
|
|
|
.if !target(do-fetch)
|
|
do-fetch: ${_ALLFILES:S/^/${DISTDIR}\//}
|
|
@${DO_NADA}
|
|
.endif
|
|
|
|
.if !target(pre-fetch)
|
|
pre-fetch:
|
|
@${DO_NADA}
|
|
.endif
|
|
|
|
.if !target(post-fetch)
|
|
post-fetch:
|
|
@${DO_NADA}
|
|
.endif
|
|
|
|
.for _file_ in ${_ALLFILES}
|
|
. if empty(PKG_RESUME_TRANSFERS:M[yY][eE][sS]) && \
|
|
exists(${DISTDIR}/${_file_})
|
|
${DISTDIR}/${_file_}:
|
|
@${DO_NADA}
|
|
. else
|
|
${DISTDIR}/${_file_}: fetch-check-interactive do-fetch-file error-check
|
|
. endif
|
|
.endfor
|
|
|
|
######################################################################
|
|
### fetch-check-interactive (PRIVATE)
|
|
######################################################################
|
|
### fetch-check-interactive is a macro target that is inserted at the
|
|
### head of a target's command list, and will check whether the fetch
|
|
### stage for this package requires user interaction to proceed.
|
|
###
|
|
.PHONY: fetch-check-interactive
|
|
fetch-check-interactive: .USEBEFORE
|
|
.if !empty(INTERACTIVE_STAGE:Mfetch) && defined(BATCH)
|
|
@${TEST} ! -f ${.TARGET} || exit 0; \
|
|
${ERROR_MSG} "The fetch stage of this package requires user interaction to download"; \
|
|
${ERROR_MSG} "the distfiles. Please fetch the distfiles manually and place them in:"; \
|
|
${ERROR_MSG} " ${_DISTDIR}"; \
|
|
if ${TEST} -n ${MASTER_SITES:Q}""; then \
|
|
${ERROR_MSG} "The distfiles are available from:"; \
|
|
for site in ${MASTER_SITES}; do \
|
|
${ERROR_MSG} " $$site"; \
|
|
done; \
|
|
fi; \
|
|
if ${TEST} -n ${HOMEPAGE:Q}""; then \
|
|
${ERROR_MSG} "See the following URL for more details:"; \
|
|
${ERROR_MSG} " "${HOMEPAGE:Q}; \
|
|
fi; \
|
|
${TOUCH} ${_INTERACTIVE_COOKIE}; \
|
|
exit 1
|
|
.elif defined(FETCH_MESSAGE) && !empty(FETCH_MESSAGE)
|
|
@${TEST} ! -f ${.TARGET} || exit 0; \
|
|
${ERROR_MSG} "======================================================================"; \
|
|
${ERROR_MSG} ""; \
|
|
for line in ${FETCH_MESSAGE}; do ${ERROR_MSG} "$$line"; done; \
|
|
${ERROR_MSG} ""; \
|
|
${ERROR_MSG} "======================================================================"; \
|
|
exit 1
|
|
.else
|
|
@${DO_NADA}
|
|
.endif
|
|
|
|
######################################################################
|
|
### do-fetch-file (PRIVATE)
|
|
######################################################################
|
|
### do-fetch-file is a macro target that runs the "fetch" script to
|
|
### transfer the files from the appropriate sites if needed.
|
|
###
|
|
#
|
|
# FETCH_CMD is the program used to fetch files. It must understand
|
|
# fetching files located via URLs, e.g. NetBSD's ftp, net/tnftp,
|
|
# etc. The default value is set in pkgsrc/mk/defaults/mk.conf.
|
|
#
|
|
# The following variables are all lists of options to pass to he command
|
|
# used to do the actual fetching of the file.
|
|
#
|
|
# FETCH_BEFORE_ARGS appear before all other options on the command line.
|
|
#
|
|
# FETCH_AFTER_ARGS appear after all other options on the command line.
|
|
#
|
|
# FETCH_RESUME_ARGS appears just after FETCH_BEFORE_ARGS and is the set
|
|
# of options for causing the command to resume a previous transfer.
|
|
#
|
|
# FETCH_OUTPUT_ARGS is a set of options for specifying the name of the
|
|
# local file that will hold the contents of the fetched file.
|
|
#
|
|
# FAILOVER_FETCH, if defined, will cause a checksum to be performed during
|
|
# a fetch to verify the transferred file is correct; if the checksum
|
|
# is incorrect, then the next site will be tried.
|
|
#
|
|
# PKG_RESUME_TRANSFERS is a yes/no variable that causes the fetch script
|
|
# to try to resume interrupted file transfers to avoid downloading
|
|
# the whole file. The default is set in pkgsrc/mk/defaults/mk.conf.
|
|
#
|
|
#FETCH_CMD?= ftp # default is set by pkgsrc/mk/defaults/mk.conf
|
|
FETCH_BEFORE_ARGS?= ${_FETCH_BEFORE_ARGS.${FETCH_CMD:T}}
|
|
FETCH_AFTER_ARGS?= ${_FETCH_AFTER_ARGS.${FETCH_CMD:T}}
|
|
FETCH_RESUME_ARGS?= ${_FETCH_RESUME_ARGS.${FETCH_CMD:T}}
|
|
FETCH_OUTPUT_ARGS?= ${_FETCH_OUTPUT_ARGS.${FETCH_CMD:T}}
|
|
|
|
_FETCH_BEFORE_ARGS.ftp= # empty
|
|
# If this host is behind a filtering firewall, use passive ftp(1)
|
|
_FETCH_BEFORE_ARGS.ftp+= ${PASSIVE_FETCH:D-p}
|
|
_FETCH_AFTER_ARGS.ftp= # empty
|
|
_FETCH_RESUME_ARGS.ftp= -R
|
|
_FETCH_OUTPUT_ARGS.ftp= -o
|
|
|
|
_FETCH_CMD= ${SETENV} CHECKSUM=${_CHECKSUM_CMD:Q} \
|
|
CP=${TOOLS_CP:Q} \
|
|
ECHO=${TOOLS_ECHO:Q} \
|
|
FETCH_CMD=${FETCH_CMD:Q} \
|
|
FETCH_BEFORE_ARGS=${FETCH_BEFORE_ARGS:Q} \
|
|
FETCH_AFTER_ARGS=${FETCH_AFTER_ARGS:Q} \
|
|
FETCH_RESUME_ARGS=${FETCH_RESUME_ARGS:Q} \
|
|
FETCH_OUTPUT_ARGS=${FETCH_OUTPUT_ARGS:Q} \
|
|
MKDIR=${TOOLS_MKDIR:Q} MV=${TOOLS_MV:Q} \
|
|
TEST=${TOOLS_TEST:Q} TOUCH=${TOOLS_TOUCH:Q} \
|
|
WC=${TOOLS_WC:Q} \
|
|
${SH} ${PKGSRCDIR}/mk/fetch/fetch
|
|
|
|
_FETCH_ARGS+= ${PKG_VERBOSE:D-v}
|
|
_FETCH_ARGS+= ${FAILOVER_FETCH:D-c}
|
|
.if exists(${DISTINFO_FILE})
|
|
_FETCH_ARGS+= -f ${DISTINFO_FILE:Q}
|
|
.endif
|
|
.if !empty(PKG_RESUME_TRANSFERS:M[yY][eE][sS])
|
|
_FETCH_ARGS+= -r
|
|
.endif
|
|
_FETCH_ARGS+= -d ${DIST_SUBDIR:U.}
|
|
|
|
.PHONY: do-fetch-file
|
|
do-fetch-file: .USE
|
|
@${STEP_MSG} "Fetching ${.TARGET:T}"
|
|
${_PKG_SILENT}${_PKG_DEBUG}${MKDIR} ${.TARGET:H}
|
|
${_PKG_SILENT}${_PKG_DEBUG} \
|
|
for d in "" ${DIST_PATH:S/:/ /g}; do \
|
|
case $$d in \
|
|
""|${DISTDIR}) continue ;; \
|
|
esac; \
|
|
file="$$d/${DIST_SUBDIR}/${.TARGET:T}"; \
|
|
if ${TEST} -f $$file; then \
|
|
${ECHO} "Using $$file"; \
|
|
${RM} -f ${.TARGET}; \
|
|
${LN} -s $$file ${.TARGET}; \
|
|
fi; \
|
|
done
|
|
${_PKG_SILENT}${_PKG_DEBUG}set -e; \
|
|
unsorted_sites="${SITES.${.TARGET:T:S/=/--/}}"; \
|
|
sites="${_ORDERED_SITES} ${_MASTER_SITE_BACKUP}"; \
|
|
cd ${.TARGET:H:S/\/${DIST_SUBDIR}$//} && \
|
|
${_FETCH_CMD} ${_FETCH_ARGS} ${.TARGET:T} $$sites
|
|
${_PKG_SILENT}${_PKG_DEBUG} \
|
|
if ${TEST} ! -f ${.TARGET}; then \
|
|
${ERROR_MSG} "Could not fetch the following file:"; \
|
|
${ERROR_MSG} " ${.TARGET:T}"; \
|
|
${ERROR_MSG} ""; \
|
|
${ERROR_MSG} "Please retrieve this file manually into:"; \
|
|
${ERROR_MSG} " ${.TARGET:H}"; \
|
|
exit 1; \
|
|
fi
|
|
|
|
######################################################################
|
|
### mirror-distfiles (PUBLIC)
|
|
######################################################################
|
|
### mirror-distfiles is a public target that is mostly of use only to
|
|
### sites that wish to provide distfiles that others may fetch. It
|
|
### only fetches distfiles that are freely re-distributable by setting
|
|
### NO_SKIP (see bsd.fetch-vars.mk).
|
|
###
|
|
.PHONY: mirror-distfiles
|
|
.if defined(NO_SRC_ON_FTP)
|
|
mirror-distfiles:
|
|
@${DO_NADA}
|
|
.else
|
|
mirror-distfiles: fetch
|
|
.endif
|
|
|
|
# XXX
|
|
# XXX Is this target actually used? (jlam 20060718)
|
|
# XXX
|
|
.PHONY: show-distfiles
|
|
.if !target(show-distfiles)
|
|
show-distfiles:
|
|
. if defined(PKG_FAIL_REASON)
|
|
${_PKG_SILENT}${_PKG_DEBUG}${DO_NADA}
|
|
. else
|
|
${_PKG_SILENT}${_PKG_DEBUG} \
|
|
for file in "" ${_CKSUMFILES}; do \
|
|
if [ "X$$file" = "X" ]; then continue; fi; \
|
|
${ECHO} $$file; \
|
|
done
|
|
. endif
|
|
.endif
|