mk: Add PKGSRC_USE_MKTOOLS support.

When enabled, pull in the pkgtools/mktools package to use C-based tools for
certain parts of the pkgsrc mk infrastructure.  Default is off for now while
any portability issues are shaken out, but it has been tested successfully on
at least illumos, macOS, and NetBSD.

The first tool is mk-buildlink-symlinks which creates the buildlink3 symlinks
as part of the "wrapper" phase.  This significantly improves performance,
especially with packages that have a lot of buildlink3 dependencies.  For
example, the time for "bmake wrapper" in x11/kde-workspace4 goes from:

  real     3:20.696394375
  user       50.553556463
  sys      2:23.883823896

to:

  real       19.873100598
  user        8.141441461
  sys        11.740602820

It's expected that other tools will be added over time to speed up other parts
of the infrastructure.

mk/bsd.pkg.use.mk

@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.pkg.use.mk,v 1.71 2022/05/09 15:35:31 nia Exp $
+#	$NetBSD: bsd.pkg.use.mk,v 1.72 2022/06/07 10:04:25 jperkin Exp $
 #
 # Turn USE_* macros into proper depedency logic.  Included near the top of
 # bsd.pkg.mk, after bsd.prefs.mk.
@@ -109,3 +109,10 @@ TOOL_DEPENDS+=		libtool-base>=${_OPSYS_LIBTOOL_REQD:U${LIBTOOL_REQD}}:../../deve
 CONFIGURE_ENV+=		LIBTOOL="${LIBTOOL} ${LIBTOOL_FLAGS}"
 MAKE_ENV+=		LIBTOOL="${LIBTOOL} ${LIBTOOL_FLAGS}"
 .endif
+
+# PKGSRC_USE_MKTOOLS.  Cyclic dependencies prevent us from using mktools
+# when building cwrappers, so use the shell tools in that instance.
+.if ${_PKGSRC_USE_MKTOOLS} == "yes" && empty(PKGPATH:Mpkgtools/cwrappers)
+TOOL_DEPENDS+=		mktools-[0-9]*:../../pkgtools/mktools
+PKG_MKSYMLINKS?=	${LOCALBASE}/libexec/mktools/mk-buildlink-symlinks
+.endif

mk/bsd.prefs.mk

@@ -1,4 +1,4 @@
-# $NetBSD: bsd.prefs.mk,v 1.421 2022/05/01 08:03:40 nia Exp $
+# $NetBSD: bsd.prefs.mk,v 1.422 2022/06/07 10:04:25 jperkin Exp $
 #
 # This file includes the mk.conf file, which contains the user settings.
 #
@@ -874,6 +874,14 @@ _USE_CWRAPPERS=		yes
 _USE_CWRAPPERS=		no
 .endif
 
+# Use C-based tools to speed up pkgsrc infrastructure tasks.
+.if empty(PKGPATH:Mpkgtools/mktools) && \
+    ${PKGSRC_USE_MKTOOLS:tl} == "yes"
+_PKGSRC_USE_MKTOOLS=	yes
+.else
+_PKGSRC_USE_MKTOOLS=	no
+.endif
+
 # Wrapper framework definitions
 .include "wrapper/wrapper-defs.mk"
 

mk/buildlink3/bsd.buildlink3.mk

@@ -1,4 +1,4 @@
-# $NetBSD: bsd.buildlink3.mk,v 1.247 2021/06/08 10:10:02 bouyer Exp $
+# $NetBSD: bsd.buildlink3.mk,v 1.248 2022/06/07 10:04:25 jperkin Exp $
 #
 # Copyright (c) 2004 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -660,7 +660,7 @@ _BLNK_FILES_CMD.${_pkg_}+=	)
 _BLNK_FILES_CMD.${_pkg_}+=	| ${SORT} -u
 
 ${_BLNK_COOKIE.${_pkg_}}:
-	${RUN}					\
+	${RUN}								\
 	case "${BUILDLINK_PREFIX.${_pkg_}}" in				\
 	*not_found)							\
 		${ERROR_MSG} "${BUILDLINK_API_DEPENDS.${_pkg_}} is not installed; can't buildlink files."; \
@@ -671,6 +671,7 @@ ${_BLNK_COOKIE.${_pkg_}}:
 		${ERROR_MSG} "[bsd.buildlink3.mk] X11BASE is not set correctly."; \
 		exit 1;							\
 	}
+.if ${_PKGSRC_USE_MKTOOLS} == "yes"
 	${RUN}								\
 	case "${BUILDLINK_PREFIX.${_pkg_}}" in				\
 	${LOCALBASE})   buildlink_dir="${BUILDLINK_DIR}" ;;		\
@@ -681,8 +682,48 @@ ${_BLNK_COOKIE.${_pkg_}}:
 	cd ${BUILDLINK_PREFIX.${_pkg_}} &&				\
 	${_BLNK_FILES_CMD.${_pkg_}} |					\
 	while read file; do						\
-		src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file";		\
-		if [ ! -f "$$src" ]; then					\
+		src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
+		[ -f "$$src" ] || continue;				\
+		dest="$$buildlink_dir/$$file";				\
+		if [ -n "${BUILDLINK_FNAME_TRANSFORM.${_pkg_}:Q}" ]; then \
+			dest=`${ECHO} $$dest | ${SED} ${BUILDLINK_FNAME_TRANSFORM.${_pkg_}}`; \
+		fi;							\
+		case "$$src" in						\
+		*.la)							\
+			dir="$${dest%/*}";				\
+			if [ ! -d "$$dir" ]; then			\
+				${MKDIR} "$$dir";			\
+			fi;						\
+			${_BLNK_LT_ARCHIVE_FILTER.${_pkg_}}		\
+				"$$src" > "$$dest";			\
+			;;						\
+		*)							\
+			${ECHO} "$$dest -> $$src" >>${.TARGET};		\
+			;;						\
+		esac;							\
+	done;								\
+	if [ -f ${.TARGET} ]; then					\
+		${PKG_MKSYMLINKS} < ${.TARGET} || {			\
+			${ECHO} "Removing ${.TARGET}";			\
+			${RM} -f ${.TARGET};				\
+			exit 1;						\
+		}							\
+	else								\
+		${TOUCH} ${TOUCH_FLAGS} ${.TARGET};			\
+	fi
+.else
+	${RUN}								\
+	case "${BUILDLINK_PREFIX.${_pkg_}}" in				\
+	${LOCALBASE})   buildlink_dir="${BUILDLINK_DIR}" ;;		\
+	${X11BASE})     buildlink_dir="${BUILDLINK_X11_DIR}" ;;		\
+	*)              buildlink_dir="${BUILDLINK_DIR}" ;;		\
+	esac;								\
+	[ -z "${BUILDLINK_PREFIX.${_pkg_}:Q}" ] ||			\
+	cd ${BUILDLINK_PREFIX.${_pkg_}} &&				\
+	${_BLNK_FILES_CMD.${_pkg_}} |					\
+	while read file; do						\
+		src="${_CROSS_DESTDIR}${BUILDLINK_PREFIX.${_pkg_}}/$$file"; \
+		if [ ! -f "$$src" ]; then				\
 			msg="$$src: not found";				\
 		else							\
 			if [ -z "${BUILDLINK_FNAME_TRANSFORM.${_pkg_}:Q}" ]; then \
@@ -713,6 +754,7 @@ ${_BLNK_COOKIE.${_pkg_}}:
 		fi;							\
 		${ECHO} "$$msg" >> ${.TARGET};				\
 	done
+.endif
 
 # _BLNK_LT_ARCHIVE_FILTER.${_pkg_} is a command-line filter used in
 # the previous target for transforming libtool archives (*.la) to

mk/defaults/mk.conf

@@ -1,4 +1,4 @@
-# $NetBSD: mk.conf,v 1.333 2022/06/07 09:35:15 wiz Exp $
+# $NetBSD: mk.conf,v 1.334 2022/06/07 10:04:25 jperkin Exp $
 #
 
 # This file provides default values for variables that may be overridden
@@ -22,6 +22,15 @@ USE_CWRAPPERS?=	auto
 # Possible: yes, no, auto
 # Default: auto (automatic detection, see above)
 
+PKGSRC_USE_MKTOOLS?=	no
+# Use C-based tools for various parts of the infrastructure instead of the
+# slower shell-based constructs.  Significantly improves performance, but
+# not enabled by default until portability across all platforms has been
+# confirmed.
+#
+# Possible: yes, no
+# Default: no
+
 USE_PKG_ADMIN_DIGEST?=	no
 # Record file digests instead of extracing the NetBSD RCS ID from files.
 #