mk/tools: correctly quote arguments in the tool wrapper log
Before, the tool arguments were written to the log as plain strings. Now the arguments are properly quoted, which makes it possible to replay the commands by copying them from the .work.log file. This only affects tools that are shell builtins (echo, true, false), get additional arguments (mkdir -p) or define a custom TOOLS_SCRIPT (pkg-config, to set an environment variable; or autotools). Tools that are symlinked to the real tool are not affected. The calls to the compiler are already properly logged since cwrappers takes care of that. This commit therefore makes the log entries for the compilers and the other tools more similar.
This commit is contained in:
rillig
parent
ace4eb9e2d
commit
81a41d0b94
5 changed files with 123 additions and 54 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# $NetBSD: create.mk,v 1.10 2019/03/24 08:40:07 rillig Exp $
|
||||
# $NetBSD: create.mk,v 1.11 2019/03/24 11:29:19 rillig Exp $
|
||||
#
|
||||
# Copyright (c) 2005, 2006 The NetBSD Foundation, Inc.
|
||||
# All rights reserved.
|
||||
|
|
@ -149,7 +149,7 @@ ${TOOLS_CMD.${_t_}}:
|
|||
if ${TEST} -n ${TOOLS_SCRIPT.${_t_}:Q}""; then \
|
||||
create=wrapper; \
|
||||
script=${TOOLS_SCRIPT.${_t_}:Q}; \
|
||||
logprefix='"set args "$$@"; shift; "'; \
|
||||
logprefix='"set args$$shquoted_args; shift; "'; \
|
||||
logmain=${TOOLS_SCRIPT.${_t_}:Q:Q}; \
|
||||
logsuffix=''; \
|
||||
elif ${TEST} -n ${TOOLS_PATH.${_t_}:Q}""; then \
|
||||
|
|
@ -158,7 +158,7 @@ ${TOOLS_CMD.${_t_}}:
|
|||
script=${TOOLS_SCRIPT_DFLT.${_t_}:Q}; \
|
||||
logprefix=''; \
|
||||
logmain=${TOOLS_PATH.${_t_}:Q:Q}\"\ \"${TOOLS_ARGS.${_t_}:Q:Q}; \
|
||||
logsuffix=' "$$*"'; \
|
||||
logsuffix='$$shquoted_args'; \
|
||||
else \
|
||||
case ${TOOLS_PATH.${_t_}:Q}"" in \
|
||||
/*) create=symlink ;; \
|
||||
|
|
@ -166,7 +166,7 @@ ${TOOLS_CMD.${_t_}}:
|
|||
script=${TOOLS_SCRIPT_DFLT.${_t_}:Q}; \
|
||||
logprefix=''; \
|
||||
logmain=${TOOLS_PATH.${_t_}:Q:Q}\"\ \"; \
|
||||
logsuffix=' "$$*"'; \
|
||||
logsuffix='$$shquoted_args'; \
|
||||
esac; \
|
||||
fi; \
|
||||
else \
|
||||
|
|
@ -175,11 +175,14 @@ ${TOOLS_CMD.${_t_}}:
|
|||
case "$$create" in \
|
||||
wrapper) \
|
||||
{ ${ECHO} '#!'${TOOLS_SHELL:Q}; \
|
||||
${ECHO} 'tools_wrapper_sed='${SED:Q:Q}; \
|
||||
${SED} -e '/^$$/d' -e '/^\#/d' ${PKGSRCDIR}/mk/tools/shquote.sh; \
|
||||
${ECHO} 'wrapperlog="$${TOOLS_WRAPPER_LOG-'${_TOOLS_WRAP_LOG:Q}'}"'; \
|
||||
${ECHO} '${ECHO} "[*] "'${.TARGET:Q}'" $$*" >> $$wrapperlog'; \
|
||||
${ECHO} 'logprefix='$$logprefix; \
|
||||
${ECHO} 'logmain='$$logmain; \
|
||||
${ECHO} "${ECHO} '<.>' \"\$$logprefix\$$logmain\"$$logsuffix >> \$$wrapperlog"; \
|
||||
${ECHO} 'shquote_args "$$@"'; \
|
||||
${ECHO} '${ECHO} "[*] "'${.TARGET:Q}'"$$shquoted_args" >> $$wrapperlog'; \
|
||||
${ECHO} 'logprefix='$$logprefix; \
|
||||
${ECHO} 'logmain='$$logmain; \
|
||||
${ECHO} "${ECHO} '<.>' \"\$$logprefix\$$logmain$$logsuffix\" >> \$$wrapperlog"; \
|
||||
${ECHO} "$$script"; \
|
||||
} > ${.TARGET:Q}; \
|
||||
${CHMOD} +x ${.TARGET:Q}; \
|
||||
|
|
|
|||
27
mk/tools/shquote.sh
Normal file
27
mk/tools/shquote.sh
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
#! /bin/sh
|
||||
# $NetBSD: shquote.sh,v 1.1 2019/03/24 11:29:19 rillig Exp $
|
||||
|
||||
# Quotes all shell meta characters from $1 and writes the result to $shquoted.
|
||||
shquote()
|
||||
{
|
||||
shquoted=$1
|
||||
case $shquoted in
|
||||
*\'*)
|
||||
shquoted=`$tools_wrapper_sed -e 's,'\'','\''\\\\'\'''\'',g' <<EOF
|
||||
$shquoted
|
||||
EOF`
|
||||
esac
|
||||
|
||||
case $shquoted in
|
||||
*[!!%+,\-./0-9:=@A-Z_a-z]*|'')
|
||||
shquoted="'$shquoted'"
|
||||
esac
|
||||
}
|
||||
|
||||
shquote_args() {
|
||||
shquoted_args=''
|
||||
for arg in "$@"; do
|
||||
shquote "$arg"
|
||||
shquoted_args="$shquoted_args $shquoted"
|
||||
done
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# $NetBSD: Makefile,v 1.14 2019/03/24 08:40:07 rillig Exp $
|
||||
# $NetBSD: Makefile,v 1.15 2019/03/24 11:29:19 rillig Exp $
|
||||
#
|
||||
|
||||
DISTNAME= # not applicable
|
||||
|
|
@ -14,7 +14,7 @@ LICENSE= 2-clause-bsd
|
|||
WRKSRC= ${WRKDIR}
|
||||
NO_CHECKSUM= yes
|
||||
PLIST_SRC= # none
|
||||
REGRESS_TESTS+= logging
|
||||
REGRESS_TESTS+= logging shquote
|
||||
REGRESS_TESTS+= awk sed sh sort tar tr
|
||||
USE_TOOLS+= awk sed sh sort tar tr
|
||||
|
||||
|
|
@ -56,10 +56,10 @@ TOOLS_PATH.path-args= echo
|
|||
TOOLS_ARGS.path-args= " \"'\\$$" "*"
|
||||
|
||||
do-build:
|
||||
.for t in ${REGRESS_TESTS}
|
||||
${RUN} cd ${WRKSRC}; \
|
||||
${ECHO_MSG} "Running testsuite "${t:Q}; \
|
||||
${SH} ${FILESDIR}/${t:Q}-test.sh
|
||||
.for test in ${REGRESS_TESTS}
|
||||
@${ECHO_MSG} "Running testsuite "${test:Q}
|
||||
${RUN} cd ${WRKSRC} \
|
||||
&& PKGSRCDIR=${PKGSRCDIR} ${SH} ${FILESDIR}/${test:Q}-test.sh
|
||||
.endfor
|
||||
|
||||
.include "../../mk/bsd.pkg.mk"
|
||||
|
|
|
|||
|
|
@ -1,14 +1,9 @@
|
|||
#! /bin/sh
|
||||
# $NetBSD: logging-test.sh,v 1.5 2019/03/24 08:40:08 rillig Exp $
|
||||
# $NetBSD: logging-test.sh,v 1.6 2019/03/24 11:29:19 rillig Exp $
|
||||
|
||||
# Up to March 2019, the command logging for the wrapped tools didn't properly
|
||||
# quote the command line arguments. This meant the logging did not reflect
|
||||
# the actual tool command line.
|
||||
#
|
||||
# As of March 2019 the logging has been fixed for tool wrappers that consist
|
||||
# only of a TOOLS_PATH.${tool} and TOOLS_ARGS.${tool}. For tools with custom
|
||||
# TOOLS_SCRIPTS it's much more difficult to do the quoting properly. See the
|
||||
# wrapper for makeinfo for a good example.
|
||||
|
||||
set -eu
|
||||
|
||||
|
|
@ -64,17 +59,15 @@ test_case "TOOLS_PATH without TOOLS_ARGS"
|
|||
# argument. This is because the echo command doesn't get any
|
||||
# additional arguments by the tool wrapper (TOOLS_ARGS.echo).
|
||||
|
||||
# TODO: To avoid unintended file expansion when replaying the
|
||||
# commands, all arguments must be shquoted.
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/echo begin * * * end
|
||||
<.> echo begin * * * end
|
||||
[*] WRKDIR/.tools/bin/echo dquot " end
|
||||
<.> echo dquot " end
|
||||
[*] WRKDIR/.tools/bin/echo squot ' end
|
||||
<.> echo squot ' end
|
||||
[*] WRKDIR/.tools/bin/echo five \\\\\ end
|
||||
<.> echo five \\\\\ end
|
||||
[*] WRKDIR/.tools/bin/echo begin '*' '*' '*' end
|
||||
<.> echo begin '*' '*' '*' end
|
||||
[*] WRKDIR/.tools/bin/echo dquot '"' end
|
||||
<.> echo dquot '"' end
|
||||
[*] WRKDIR/.tools/bin/echo squot ''\''' end
|
||||
<.> echo squot ''\''' end
|
||||
[*] WRKDIR/.tools/bin/echo five '\\\\\' end
|
||||
<.> echo five '\\\\\' end
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -84,13 +77,12 @@ test_case "TOOLS_PATH with TOOLS_ARGS"
|
|||
|
||||
run_tool mkdir "directory with spaces"
|
||||
|
||||
# The log doesn't show delimiters for the arguments, which makes
|
||||
# the call to mkdir ambiguous. Doing proper shell quoting would
|
||||
# require code similar to shquote from mk/scripts/shell-lib.
|
||||
# This may make the tools wrapper slower.
|
||||
# The TOOLS_ARGS are already quoted, therefore they all look
|
||||
# different in the log. The actual arguments, on the other hand,
|
||||
# all look the same.
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/mkdir directory with spaces
|
||||
<.> BINDIR/mkdir -p directory with spaces
|
||||
[*] WRKDIR/.tools/bin/mkdir 'directory with spaces'
|
||||
<.> BINDIR/mkdir -p 'directory with spaces'
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -99,18 +91,18 @@ test_case "TOOLS_PATH with TOOLS_ARGS containing double quotes"
|
|||
run_tool path-args-dquot "and" \" "\"" '"'
|
||||
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/path-args-dquot and " " "
|
||||
<.> echo \" "\"" '"' and " " "
|
||||
[*] WRKDIR/.tools/bin/path-args-dquot and '"' '"' '"'
|
||||
<.> echo \" "\"" '"' and '"' '"' '"'
|
||||
EOF
|
||||
}
|
||||
|
||||
test_case "TOOLS_PATH with TOOLS_ARGS containing special characters"
|
||||
{
|
||||
run_tool path-args "and" " \"'\\$" "*"
|
||||
run_tool path-args "and" " \"'\\\$" "*"
|
||||
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/path-args and "'\$ *
|
||||
<.> echo " \"'\\$" "*" and "'\$ *
|
||||
[*] WRKDIR/.tools/bin/path-args and ' "'\''\$' '*'
|
||||
<.> echo " \"'\\$" "*" and ' "'\''\$' '*'
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -118,11 +110,9 @@ test_case "TOOLS_SCRIPT with dquot"
|
|||
{
|
||||
run_tool script-dquot
|
||||
|
||||
# The following log output contains a trailing whitespace. This
|
||||
# is because the tool didn't get any actual arguments.
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/script-dquot
|
||||
<.> set args ; shift; echo "hello; world"
|
||||
[*] WRKDIR/.tools/bin/script-dquot
|
||||
<.> set args; shift; echo "hello; world"
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -130,11 +120,9 @@ test_case "TOOLS_SCRIPT with backslashes"
|
|||
{
|
||||
run_tool script-backslash
|
||||
|
||||
# The following log output contains a trailing whitespace. This
|
||||
# is because the tool didn't get any actual arguments.
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/script-backslash
|
||||
<.> set args ; shift; echo hello\;\ world
|
||||
[*] WRKDIR/.tools/bin/script-backslash
|
||||
<.> set args; shift; echo hello\;\ world
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -149,8 +137,6 @@ test_case "TOOLS_SCRIPT with complicated replacement"
|
|||
#
|
||||
# In this example, the $0 becomes unrealistic when the command
|
||||
# is replayed. In practice $0 is seldom used.
|
||||
#
|
||||
# TODO: In the "set arg", the arguments must be shquoted.
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/for-loop one two three
|
||||
<.> set args one two three; shift; printf '%s' "$0"; for arg in "$@"; do printf ' <%s>' "$arg"; done; printf '\n'
|
||||
|
|
@ -165,9 +151,8 @@ test_case "TOOLS_SCRIPT with actual arguments containing quotes"
|
|||
-DDD="\"a b\"" \
|
||||
-DB=\"a\ b\"
|
||||
|
||||
# TODO: Add proper quoting for the arguments.
|
||||
assert_log <<'EOF'
|
||||
[*] WRKDIR/.tools/bin/for-loop -DSD="a b" -DSS='a b' -DDD="a b" -DB="a b"
|
||||
<.> set args -DSD="a b" -DSS='a b' -DDD="a b" -DB="a b"; shift; printf '%s' "$0"; for arg in "$@"; do printf ' <%s>' "$arg"; done; printf '\n'
|
||||
[*] WRKDIR/.tools/bin/for-loop '-DSD="a b"' '-DSS='\''a b'\''' '-DDD="a b"' '-DB="a b"'
|
||||
<.> set args '-DSD="a b"' '-DSS='\''a b'\''' '-DDD="a b"' '-DB="a b"'; shift; printf '%s' "$0"; for arg in "$@"; do printf ' <%s>' "$arg"; done; printf '\n'
|
||||
EOF
|
||||
}
|
||||
|
|
|
|||
54
regress/tools/files/shquote-test.sh
Normal file
54
regress/tools/files/shquote-test.sh
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
#! /bin/sh
|
||||
set -eu
|
||||
|
||||
. "${PKGSRCDIR}/mk/tools/shquote.sh"
|
||||
|
||||
# test_shquote $in becomes $out
|
||||
test_shquote() {
|
||||
shquote "$1"
|
||||
[ "x$shquoted" = "x$3" ] && return
|
||||
printf 'input: %s\nexpected: %s\nactual: %s\n' "$1" "$3" "$shquoted" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
tools_wrapper_sed=${SED:-/usr/bin/sed}
|
||||
|
||||
test_shquote '' becomes "''"
|
||||
|
||||
test_shquote ' ' becomes "' '"
|
||||
test_shquote '!' becomes '!'
|
||||
test_shquote '"' becomes \'\"\'
|
||||
test_shquote '#' becomes "'#'"
|
||||
test_shquote '$' becomes "'$'"
|
||||
test_shquote '%' becomes '%'
|
||||
test_shquote '&' becomes "'&'"
|
||||
test_shquote \' becomes "''\\'''"
|
||||
test_shquote '(' becomes "'('"
|
||||
test_shquote ')' becomes "')'"
|
||||
test_shquote '*' becomes "'*'"
|
||||
test_shquote '+,-./0123456789:' becomes '+,-./0123456789:'
|
||||
test_shquote ';' becomes "';'"
|
||||
test_shquote '<' becomes "'<'"
|
||||
test_shquote '=' becomes '='
|
||||
test_shquote '>' becomes "'>'"
|
||||
test_shquote '?' becomes "'?'"
|
||||
test_shquote '@ABCDEFGHIJKLMNOPQRSTUVWXYZ' becomes '@ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
test_shquote '[' becomes "'['"
|
||||
test_shquote '\' becomes \'\\\'
|
||||
test_shquote ']' becomes "']'"
|
||||
test_shquote '^' becomes "'^'"
|
||||
test_shquote '_' becomes '_'
|
||||
test_shquote '`' becomes "'\`'"
|
||||
test_shquote 'abcdefghijklmnopqrstuvwxyz' becomes 'abcdefghijklmnopqrstuvwxyz'
|
||||
test_shquote '{' becomes "'{'"
|
||||
test_shquote '|' becomes "'|'"
|
||||
test_shquote '}' becomes "'}'"
|
||||
test_shquote '~' becomes "'~'"
|
||||
|
||||
test_shquote 'a b' becomes "'a b'"
|
||||
test_shquote 'a b' becomes "'a b'"
|
||||
test_shquote ' ' becomes "' '"
|
||||
test_shquote '-e asdf' becomes "'-e asdf'"
|
||||
test_shquote '-n' becomes '-n'
|
||||
test_shquote '\\\\\\\\' becomes \''\\\\\\\\'\'
|
||||
test_shquote \"\$\'\;\<\\\` becomes \'\"\$\'\\\'\'\;\<\\\`\'
|
||||
Loading…
Reference in a new issue