Refactor dependency checks and installation

This is an important step to prepare the ports tree for VARIANTS(aka flavours)
and subpackage by making the dependency code easier to deal with.

Change:
- Externalize in a proper shell script the code that was an inlined shell script
- Add better validation on the syntaxe used
- test after the dependency has been installed that it actually really fulfill
  the pattern searched (improving QA)
- Unify lib-depends with other dependency checks
- Make ${PORTSDIR} not mandatory anymore in _DEPENDS lines:
  aka pattern:${PORTSDIR}/category/port can now be written pattern:category/port
  /!\ Please to not use this syntax yet! poudriere have received a fix to be
  able to handle this new syntax (but no new release of poudriere has it yet)
  portmaster/portupgrade hasn't been checked. if one cares about those last 2 it
  would be really nice to provide patches to them!
- Remove _DEPENDS_ALWAYS it has half broken for a while and did not really make
  sense.
- Keep STRICT_DEPENDS for now it might not be necessary anymore given all the
  new checks added, but until someone confirms it is worth keeping it.

Note that all the env passed are prefixed by 'dp_' to avoid polluting children
make

Differential Revision:	https://reviews.freebsd.org/D2897
Reviewed by:	antoine
Exp-run by:	antoine

Mk/Scripts/do-depends.sh

@@ -0,0 +1,177 @@
+#!/bin/sh
+# $FreeBSD$
+#
+# MAINTAINER: portmgr@FreeBSD.org
+
+set -e
+
+. ${dp_SCRIPTSDIR}/functions.sh
+
+envfault=
+for i in dp_RAWDEPENDS dp_DEPTYPE dp_DEPENDS_TARGET dp_DEPENDS_PRECLEAN \
+	dp_DEPENDS_CLEAN dp_DEPENDS_ARGS dp_USE_PACKAGE_DEPENDS \
+	dp_USE_PACKAGE_DEPENDS_ONLY dp_PKG_ADD dp_PKG_INFO dp_WRKDIR \
+	dp_PKGNAME dp_STRICT_DEPENDS dp_LOCALBASE dp_LIB_DIRS dp_SH \
+	dp_SCRIPTSDIR dp_PORTSDIR dp_MAKE
+do
+	if ! (eval ": \${${i}?}" ) >/dev/null; then
+		envfault="${envfault}${envfault:+" "}${i}"
+	fi
+done
+if [ -n "${envfault}" ]; then
+	echo "Environment variable ${envfault} undefined. Aborting." \
+		| fmt >&2
+	exit 1
+fi
+
+set -u
+
+install_depends()
+{
+	origin=$1
+	target=$2
+	depends_args=$3
+	if [ -z "${dp_USE_PACKAGE_DEPENDS}" -a -z "${dp_USE_PACKAGE_DEPENDS_ONLY}" ]; then
+		${dp_MAKE} -C ${origin} -DINSTALLS_DEPENDS ${target} ${depends_args}
+		return 0
+	fi
+
+	read pkgfile <<- EOF
+	$(${dp_MAKE} -C ${origin} -VPKGFILE)
+	EOF
+	read pkgbase <<- EOF
+	$(${dp_MAKE} -C ${origin} -VPKGBASE)
+	EOF
+	if [ -r "${pkgfile}" -a "${target}" = "${dp_DEPENDS_TARGET}" ]; then
+		echo "===>   Installing existing package ${pkgfile}"
+		if [ "${pkgbase}" = "pkg" ]; then
+			[ -d ${dp_WRKDIR} ] || mkdir -p ${dp_WRKDIR}
+			tar xf ${pkgfile} -C ${dp_WRKDIR} -s ",/.*/,,g" "*/pkg-static"
+			${dp_WRKDIR}/pkg-static add ${pkgfile}
+			rm -f ${dp_WRKDIR}/pkg-static
+		else
+			${dp_PKG_ADD} -A ${pkgfile}
+		fi
+	elif [ -n "${dp_USE_PACKAGE_DEPENDS_ONLY}" -a "${target}" = "${dp_DEPENDS_TARGET}" ]; then
+		echo "===>   ${dp_PKGNAME} depends on package: ${pkgfile} - not found" >&2
+		echo "===>   dp_USE_PACKAGE_DEPENDS_ONLY set - not building missing dependency from source" >&2
+		exit 1
+	else
+		${dp_MAKE} -C ${origin} -DINSTALLS_DEPENDS ${target} ${depends_args}
+	fi
+}
+
+find_package()
+{
+	if ${dp_PKG_INFO} "$1" >/dev/null 2>&1; then
+		echo "===>   ${dp_PKGNAME} depends on package: $1 - found"
+		return 0
+	fi
+	echo "===>   ${dp_PKGNAME} depends on file: $1 - not found"
+	return 1
+}
+
+find_file()
+{
+	if [ -e "$1" ]; then
+		echo "===>   ${dp_PKGNAME} depends on file: $1 - found"
+		return 0
+	fi
+	echo "===>   ${dp_PKGNAME} depends on file: $1 - not found"
+	return 1
+}
+
+find_file_path()
+{
+	if which -s $1 ; then
+		echo "===>   ${dp_PKGNAME} depends on executable: $1 - found"
+		return 0
+	fi
+	echo "===>   ${dp_PKGNAME} depends on executable: $1 - not found"
+	return 1
+}
+
+find_lib()
+{
+	echo -n "===>   ${dp_PKGNAME} depends on shared library: $1"
+	libfile=$(env -i LIB_DIRS="${dp_LIB_DIRS}" LOCALBASE="${dp_LOCALBASE}" ${dp_SH} ${dp_SCRIPTSDIR}/find-lib.sh $1)
+	if [ -z "${libfile}" ]; then
+		echo " - not found"
+		return 1
+	fi
+	echo " - found (${libfile})"
+}
+
+anynotfound=0
+for _line in ${dp_RAWDEPENDS} ; do
+	myifs=${IFS}
+	IFS=:
+	set -- ${_line}
+	IFS=${myifs}
+	if [ $# -lt 2 -o $# -gt 3 ]; then
+		echo "Error: bad dependency syntax in ${dp_DEPTYPE}" >&2
+		echo "expecting: pattern:origin[:target]" >&2
+		echo "got: ${_line}" >&2
+		exit 1
+	fi
+	pattern=$1
+	origin=$2
+	last=${3:-}
+
+	if [ -z "${pattern}" ]; then
+		echo "Error: there is an empty port dependency in ${dp_DEPTYPE}" >&2
+		exit 1
+	fi
+
+	if [ -z "${origin}" ]; then
+		echo "Error: a dependency has an empty origin in ${dp_DEPTYPE}" >&2
+		exit 1
+	fi
+
+	case "${origin}" in
+	/*) ;;
+	*) origin="${dp_PORTSDIR}/${origin}" ;;
+	esac
+	if [ ! -f "${origin}/Makefile" ]; then
+		echo "Error a dependency refers to a non existing origin: ${origin} in ${dp_DEPTYPE}" >&2
+		exit 1
+	fi
+
+	depends_args="${dp_DEPENDS_ARGS}"
+	target=${dp_DEPENDS_TARGET}
+	if [ -n "${last}" ]; then
+		target=${last}
+		if [ -n "${dp_DEPENDS_PRECLEAN}" ]; then
+			target="clean ${target}"
+			depends_args="NOCLEANDEPENDS=yes"
+		fi
+		if [ -n "${dp_DEPENDS_CLEAN}" ]; then
+			target="${target} clean"
+			depends_args="NOCLEANDEPENDS=yes"
+		fi
+	fi
+
+	case ${pattern} in
+	*\>*|*\<*|*=*) fct=find_package ;;
+	lib*.so*)      fct=find_lib ;;
+	/nonexistent)  fct=false ;;
+	/*)            fct=find_file ;;
+	*)             fct=find_file_path ;;
+	esac
+	if ${fct} "${pattern}" ; then
+		continue
+	fi
+	[ ${pattern} = "/nonexistent" ] || anynotfound=1
+
+	# Now actually install the dependencies
+	install_depends "${origin}" "${target}" "${depends_args}"
+	# Recheck if the installed dependency validates the pattern except for /nonexistent
+	[ "${fct}" = "false" ] || ${fct} "${pattern}"
+	echo "===>   Returning to build of ${dp_PKGNAME}"
+done
+
+if [ -n "${dp_STRICT_DEPENDS}" -a ${anynotfound} -eq 1 ]; then \
+	echo "===>   dp_STRICT_DEPENDS set - Not installing missing dependencies."
+	echo "       This means a dependency is wrong since it was not satisfied in the ${dp_DEPTYPE} phase."
+	exit 1
+fi

Mk/bsd.port.mk

@@ -4329,177 +4329,33 @@ package-noinstall: package
 .if !target(depends)
 depends: pkg-depends extract-depends patch-depends lib-depends fetch-depends build-depends run-depends
 
-.if defined(ALWAYS_BUILD_DEPENDS)
-_DEPEND_ALWAYS=	1
-.else
-_DEPEND_ALWAYS=	0
-.endif
-
-_INSTALL_DEPENDS=	\
-		if [ -n "${USE_PACKAGE_DEPENDS}" -o -n "${USE_PACKAGE_DEPENDS_ONLY}" ]; then \
-			subpkgfile=`(cd $$dir; ${MAKE} $$depends_args -V PKGFILE)`; \
-			subpkgname=$${subpkgfile%-*} ; \
-			subpkgname=$${subpkgname\#\#*/} ; \
-			if [ -r "$${subpkgfile}" -a "$$target" = "${DEPENDS_TARGET}" ]; then \
-				${ECHO_MSG} "===>   Installing existing package $${subpkgfile}"; \
-				if [ $${subpkgname} = "pkg" ]; then \
-					[ -d ${WRKDIR} ] || ${MKDIR} ${WRKDIR} ; \
-					${TAR} xf $${subpkgfile} -C ${WRKDIR} -s ",/.*/,,g" "*/pkg-static" ; \
-					${WRKDIR}/pkg-static add $${subpkgfile}; \
-					${RM} -f ${WRKDIR}/pkg-static; \
-				else \
-					${PKG_ADD} -A $${subpkgfile}; \
-				fi; \
-			elif [ -n "${USE_PACKAGE_DEPENDS_ONLY}" -a "$${target}" = "${DEPENDS_TARGET}" ]; then \
-				${ECHO_MSG} "===>   ${PKGNAME} depends on package: $${subpkgfile} - not found"; \
-				${ECHO_MSG} "===>   USE_PACKAGE_DEPENDS_ONLY set - not building missing dependency from source"; \
-				exit 1; \
-			else \
-			  (cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) ; \
-			fi; \
-		elif [ -z "${STRICT_DEPENDS}" ]; then \
-			(cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) ; \
-		fi; \
-		${ECHO_MSG} "===>   Returning to build of ${PKGNAME}";
-
-.for deptype in PKG EXTRACT PATCH FETCH BUILD RUN
+.for deptype in PKG EXTRACT PATCH FETCH BUILD LIB RUN
 ${deptype:tl}-depends:
-.if defined(${deptype}_DEPENDS)
-.if !defined(NO_DEPENDS)
-	@set -e ; anynotfound=0; for i in `${ECHO_CMD} "${${deptype}_DEPENDS}"`; do \
-		prog=$${i%%:*}; \
-		if [ -z "$$prog" ]; then \
-			${ECHO_MSG} "Error: there is an empty port dependency in ${deptype}_DEPENDS."; \
-			break; \
-		fi; \
-		dir=`${ECHO_CMD} $$i | ${SED} -e 's/[^:]*://'`; \
-		if ${EXPR} "$$dir" : '.*:' > /dev/null; then \
-			target=$${dir##*:}; \
-			dir=$${dir%%:*}; \
-			if [ X${DEPENDS_PRECLEAN} != "X" ]; then \
-				target="clean $$target"; \
-				depends_args="$$depends_args NOCLEANDEPENDS=yes"; \
-			fi; \
-			if [ X${DEPENDS_CLEAN} != "X" ]; then \
-				target="$$target clean"; \
-				depends_args="$$depends_args NOCLEANDEPENDS=yes"; \
-			fi; \
-		else \
-			target="${DEPENDS_TARGET}"; \
-			depends_args="${DEPENDS_ARGS}"; \
-		fi; \
-		if ${EXPR} "$$prog" : \\/ >/dev/null; then \
-			if [ -e "$$prog" ]; then \
-				if [ "$$prog" = "${NONEXISTENT}" ]; then \
-					${ECHO_MSG} "Error: ${NONEXISTENT} exists.  Please remove it, and restart the build."; \
-					${FALSE}; \
-				else \
-					${ECHO_MSG} "===>   ${PKGNAME} depends on file: $$prog - found"; \
-					if [ ${_DEPEND_ALWAYS} = 1 ]; then \
-						${ECHO_MSG} "       (but building it anyway)"; \
-						notfound=1; \
-					else \
-						notfound=0; \
-					fi; \
-				fi; \
-			else \
-				${ECHO_MSG} "===>   ${PKGNAME} depends on file: $$prog - not found"; \
-				notfound=1; \
-			fi; \
-		else \
-			case $${prog} in \
-				*\>*|*\<*|*=*)	pkg=yes;; \
-				*)		pkg="";; \
-			esac; \
-			if [ "$$pkg" != "" ]; then \
-				if ${PKG_INFO} "$$prog" > /dev/null 2>&1 ; then \
-					${ECHO_MSG} "===>   ${PKGNAME} depends on package: $$prog - found"; \
-					if [ ${_DEPEND_ALWAYS} = 1 ]; then \
-						${ECHO_MSG} "       (but building it anyway)"; \
-						notfound=1; \
-					else \
-						notfound=0; \
-					fi; \
-				else \
-					${ECHO_MSG} "===>   ${PKGNAME} depends on package: $$prog - not found"; \
-					notfound=1; \
-				fi; \
-				if [ $$notfound != 0 ]; then \
-					inverse_dep=`${ECHO_CMD} $$prog | ${SED} \
-						-e 's/<=/=gt=/; s/</=ge=/; s/>=/=lt=/; s/>/=le=/' \
-						-e 's/=gt=/>/; s/=ge=/>=/; s/=lt=/</; s/=le=/<=/'`; \
-					pkg_info=`${PKG_INFO} -E "$$inverse_dep" 2>/dev/null || ${TRUE}`; \
-					if [ "$$pkg_info" != "" ]; then \
-						${ECHO_MSG} "===>   Found $$pkg_info, but you need to upgrade to $$prog."; \
-						exit 1; \
-					fi; \
-				fi; \
-			elif ${WHICH} "$$prog" > /dev/null 2>&1 ; then \
-				${ECHO_MSG} "===>   ${PKGNAME} depends on executable: $$prog - found"; \
-				if [ ${_DEPEND_ALWAYS} = 1 ]; then \
-					${ECHO_MSG} "       (but building it anyway)"; \
-					notfound=1; \
-				else \
-					notfound=0; \
-				fi; \
-			else \
-				${ECHO_MSG} "===>   ${PKGNAME} depends on executable: $$prog - not found"; \
-				notfound=1; \
-			fi; \
-		fi; \
-		if [ $$notfound != 0 ]; then \
-			if [ "$$prog" != "${NONEXISTENT}" ]; then \
-				anynotfound=1; \
-			fi; \
-			${ECHO_MSG} "===>    Verifying $$target for $$prog in $$dir"; \
-			if [ ! -d "$$dir" ]; then \
-				${ECHO_MSG} "     => No directory for $$prog.  Skipping.."; \
-			else \
-				${_INSTALL_DEPENDS} \
-			fi; \
-		fi; \
-	done; \
-	if [ -n "${STRICT_DEPENDS}" -a $${anynotfound} -eq 1 ]; then \
-		${ECHO_MSG} "===>   STRICT_DEPENDS set - Not installing missing dependencies."; \
-		${ECHO_MSG} "       This means a dependency is wrong since it was not satisfied in the ${deptype:tl}-depends phase."; \
-		exit 1; \
-	fi
-.endif
-.else
-	@${DO_NADA}
+.if defined(${deptype}_DEPENDS) && !defined(NO_DEPENDS)
+	@${SETENV} \
+		dp_RAWDEPENDS="${${deptype}_DEPENDS}" \
+		dp_DEPTYPE="${deptype}_DEPENDS" \
+		dp_DEPENDS_TARGET="${DEPENDS_TARGET}" \
+		dp_DEPENDS_PRECLEAN="${DEPENDS_PRECLEAN}" \
+		dp_DEPENDS_CLEAN="${DEPENDS_CLEAN}" \
+		dp_DEPENDS_ARGS="${DEPENDS_ARGS}" \
+		dp_USE_PACKAGE_DEPENDS="${USE_PACKAGE_DEPENDS}" \
+		dp_USE_PACKAGE_DEPENDS_ONLY="${USE_PACKAGE_DEPENDS_ONLY}" \
+		dp_PKG_ADD="${PKG_ADD}" \
+		dp_PKG_INFO="${PKG_INFO}" \
+		dp_WRKDIR="${WRKDIR}" \
+		dp_PKGNAME="${PKGNAME}" \
+		dp_STRICT_DEPENDS="${STRICT_DEPENDS}" \
+		dp_LOCALBASE="${LOCALBASE}" \
+		dp_LIB_DIRS="${LIB_DIRS}" \
+		dp_SH="${SH}" \
+		dp_SCRIPTSDIR="${SCRIPTSDIR}" \
+		dp_PORTSDIR="${PORTSDIR}" \
+		dp_MAKE="${MAKE}" \
+		${SH} ${SCRIPTSDIR}/do-depends.sh
 .endif
 .endfor
 
-lib-depends:
-.if defined(LIB_DEPENDS) && !defined(NO_DEPENDS)
-	@set -e ; \
-	anynotfound=0; for i in ${LIB_DEPENDS}; do \
-		lib=$${i%%:*} ; \
-		dir=$${i#*:}  ; \
-		target="${DEPENDS_TARGET}"; \
-		depends_args="${DEPENDS_ARGS}"; \
-		${ECHO_MSG}  -n "===>   ${PKGNAME} depends on shared library: $${lib}" ; \
-		libfile=`${SETENV} LIB_DIRS="${LIB_DIRS}" LOCALBASE="${LOCALBASE}" ${SH} ${SCRIPTSDIR}/find-lib.sh $${lib}` ; \
-		if [ -z "$${libfile}" ]; then \
-			anynotfound=1; \
-			${ECHO_MSG} " - not found"; \
-			${ECHO_MSG} "===>    Verifying for $$lib in $$dir"; \
-			if [ ! -d "$$dir" ] ; then \
-				${ECHO_MSG} "    => No directory for $$lib.  Skipping.."; \
-			else \
-				${_INSTALL_DEPENDS} \
-			fi ; \
-		else \
-			${ECHO_MSG} " - found ($${libfile})"; \
-		fi ; \
-	done; \
-	if [ -n "${STRICT_DEPENDS}" -a $${anynotfound} -eq 1 ]; then \
-		${ECHO_MSG} "===>   STRICT_DEPENDS set - Not installing missing dependencies."; \
-		${ECHO_MSG} "       This means a dependency is wrong since it was not satisfied in the lib-depends phase."; \
-		exit 1; \
-	fi
-.endif
-
 .endif
 
 # Dependency lists: both build and runtime, recursive.  Print out directory names.