pkgsrc/pkgtools/pkglint/files/mklines.go
rillig d42d69e1a0 pkgtools/pkglint: update to 21.2.4
Changes since 21.2.3:

Fixed loading of the tool definitions.  This adds 76 warnings for
packages that use tools without adding them to USE_TOOLS.  It also fixes
the warning about gmake and Meson.
2021-08-14 08:19:49 +00:00

685 lines
19 KiB
Go

package pkglint
import "strings"
// MkLines contains data for the Makefile (or *.mk) that is currently checked.
type MkLines struct {
mklines []*MkLine
lines *Lines
// The package that provides further context for cross-checks,
// such as the conditionally included files.
//
// This package should be used mostly as a read-only storage of context
// information. To keep the code understandable, only few things should
// be changed, if at all. This is exactly the reason that the
// extraScope has been moved to a separate variable.
//
// XXX: Maybe split this field into two: pkg and pkgForModification.
pkg *Package
// The extra scope in which all variable assignments are recorded.
// In most cases this is nil.
//
// When loading the package Makefile with all its included files,
// it is set to pkg.vars.
extraScope *Scope
allVars Scope // The variables after loading the complete file
buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it.
plistVarAdded map[string]*MkLine // Identifiers that are added to PLIST_VARS.
plistVarSet map[string]*MkLine // Identifiers for which PLIST.${id} is defined.
plistVarSkip bool // True if any of the PLIST_VARS identifiers refers to a variable.
Tools *Tools // Tools defined in file scope.
indentation *Indentation // Indentation depth of preprocessing directives; only available during MkLines.ForEach.
once Once
// TODO: Consider extracting plistVarAdded, plistVarSet, plistVarSkip into an own type.
// TODO: Describe where each of the above fields is valid.
checkAllData mklinesCheckAll
}
// mklinesCheckAll contains the data that may only be accessed during a call
// to MkLines.checkAll.
type mklinesCheckAll struct {
// Current make(1) target
target string
vars Scope
// The variables currently used in .for loops
forVars map[string]bool
// Custom action that is run after checking each line
postLine func(mkline *MkLine)
}
func NewMkLines(lines *Lines, pkg *Package, extraScope *Scope) *MkLines {
mklines := make([]*MkLine, lines.Len())
for i, line := range lines.Lines {
mklines[i] = NewMkLineParser().Parse(line)
}
tools := NewTools()
tools.Fallback(G.Pkgsrc.Tools)
return &MkLines{
mklines,
lines,
pkg,
extraScope,
NewScope(),
make(map[string]bool),
make(map[string]*MkLine),
make(map[string]*MkLine),
false,
tools,
nil,
Once{},
mklinesCheckAll{
target: "",
vars: NewScope(),
forVars: make(map[string]bool),
postLine: nil}}
}
// TODO: Consider defining an interface MkLinesChecker (different name, though, since this one confuses even me)
// that checks a single topic, like:
//
// * PlistVars
// * ForLoops
// * MakeTargets
// * Tools
// * Indentation
// * LoadTimeVarUse
// * Subst
// * VarAlign
//
// These could be run in parallel to get the diagnostics strictly from top to bottom.
// Some of the checkers will probably depend on one another.
//
// The driving code for these checkers could look like:
//
// ck.Init
// ck.BeforeLine
// ck.Line
// ck.AfterLine
// ck.Finish
func (mklines *MkLines) Check() {
if trace.Tracing {
defer trace.Call(mklines.lines.Filename)()
}
// In the first pass, all additions to BUILD_DEFS and USE_TOOLS
// are collected to make the order of the definitions irrelevant.
mklines.collectRationale()
mklines.collectUsedVariables()
mklines.collectVariables(false, true)
mklines.collectPlistVars()
if mklines.pkg != nil {
mklines.pkg.collectConditionalIncludes(mklines)
}
// In the second pass, the actual checks are done.
mklines.checkAll()
SaveAutofixChanges(mklines.lines)
}
func (mklines *MkLines) collectRationale() {
isUseful := func(mkline *MkLine) bool {
comment := trimHspace(mkline.Comment())
return comment != "" && !hasPrefix(comment, "$NetBSD")
}
isRealComment := func(mkline *MkLine) bool {
return mkline.IsComment() && !mkline.IsCommentedVarassign()
}
var rat strings.Builder
for _, mkline := range mklines.mklines {
if isRealComment(mkline) && isUseful(mkline) {
rat.WriteString(mkline.Comment())
rat.WriteString("\n")
}
var lineRat strings.Builder
lineRat.WriteString(rat.String())
if isUseful(mkline) {
lineRat.WriteString(mkline.Comment())
lineRat.WriteString("\n")
}
mkline.splitResult.rationale = lineRat.String()
if mkline.IsEmpty() {
rat.Reset()
}
}
}
func (mklines *MkLines) collectUsedVariables() {
for _, mkline := range mklines.mklines {
mkline.ForEachUsed(func(varUse *MkVarUse, time VucTime) {
mklines.UseVar(mkline, varUse.varname, time)
})
}
mklines.collectDocumentedVariables()
}
// UseVar remembers that the given variable is used in the given line.
// This controls the "defined but not used" warning.
func (mklines *MkLines) UseVar(mkline *MkLine, varname string, time VucTime) {
mklines.allVars.Use(varname, mkline, time)
if mklines.extraScope != nil {
mklines.extraScope.Use(varname, mkline, time)
}
}
// collectDocumentedVariables collects the variables that are mentioned in the human-readable
// documentation of the Makefile fragments from the pkgsrc infrastructure.
//
// Loosely based on mk/help/help.awk, revision 1.28, but much simpler.
func (mklines *MkLines) collectDocumentedVariables() {
scope := NewScope()
commentLines := 0
relevant := true
// TODO: Correctly interpret declarations like "package-settable variables:" and
// "user-settable variables", as well as "default: ...", "allowed: ...",
// "list of" and other types.
finish := func() {
// The commentLines include the the line containing the variable name,
// leaving 2 of these 3 lines for the actual documentation.
if commentLines >= 3 && relevant {
scope.forEach(func(varname string, data *scopeVar) {
mklines.allVars.Define(varname, data.used)
mklines.allVars.Use(varname, data.used, VucRunTime)
})
}
scope = NewScope()
commentLines = 0
relevant = true
}
for _, mkline := range mklines.mklines {
text := mkline.Text
switch {
case hasPrefix(text, "#"):
words := strings.Fields(text)
if len(words) <= 1 {
break
}
commentLines++
parser := NewMkLexer(words[1], nil)
varname := parser.Varname()
if len(varname) < 3 {
break
}
if hasSuffix(varname, ".") {
if !parser.lexer.SkipRegexp(regcomp(`^<\w+>`)) {
break
}
varname += "*"
}
parser.lexer.SkipByte(':')
varcanon := varnameCanon(varname)
if varcanon == strings.ToUpper(varcanon) && matches(varcanon, `[A-Z]`) && parser.EOF() {
scope.Define(varcanon, mkline)
scope.Use(varcanon, mkline, VucRunTime)
}
if words[1] == "Copyright" {
relevant = false
}
case mkline.IsEmpty():
finish()
}
}
finish()
}
func (mklines *MkLines) collectVariables(infrastructure bool, addToUseTools bool) {
mklines.ForEach(func(mkline *MkLine) {
mklines.collectVariable(mkline, infrastructure, addToUseTools)
})
}
func (mklines *MkLines) collectVariable(mkline *MkLine, infrastructure bool, addToUseTools bool) {
mklines.Tools.ParseToolLine(mklines, mkline, infrastructure, addToUseTools)
if !mkline.IsVarassignMaybeCommented() {
return
}
mklines.defineVar(mkline, mkline.Varname())
varcanon := mkline.Varcanon()
switch varcanon {
case
"BUILD_DEFS",
"PKG_GROUPS_VARS", // see mk/misc/unprivileged.mk
"PKG_USERS_VARS": // see mk/misc/unprivileged.mk
for _, varname := range mkline.Fields() {
mklines.buildDefs[varname] = true
if trace.Tracing {
trace.Step1("%q is added to BUILD_DEFS.", varname)
}
}
case
"BUILTIN_FIND_FILES_VAR",
"BUILTIN_FIND_HEADERS_VAR":
for _, varname := range mkline.Fields() {
mklines.allVars.Define(varname, mkline)
}
case "PLIST_VARS":
for _, id := range mkline.ValueFields(resolveVariableRefs(mkline.Value(), mklines, nil)) {
if trace.Tracing {
trace.Step1("PLIST.%s is added to PLIST_VARS.", id)
}
if containsVarUse(id) {
mklines.UseVar(mkline, "PLIST.*", mkline.Op().Time())
mklines.plistVarSkip = true
} else {
mklines.UseVar(mkline, "PLIST."+id, mkline.Op().Time())
}
}
case "SUBST_VARS.*":
for _, substVar := range mkline.Fields() {
mklines.UseVar(mkline, varnameCanon(substVar), mkline.Op().Time())
if trace.Tracing {
trace.Step1("varuse %s", substVar)
}
}
case "OPSYSVARS":
for _, opsysVar := range mkline.Fields() {
mklines.UseVar(mkline, opsysVar+".*", mkline.Op().Time())
mklines.defineVar(mkline, opsysVar)
}
}
}
// ForEach calls the action for each line, until the action returns false.
// It keeps track of the indentation (see MkLines.indentation)
// and all conditional variables (see Indentation.IsConditional).
func (mklines *MkLines) ForEach(action func(mkline *MkLine)) {
mklines.ForEachEnd(
func(mkline *MkLine) bool { action(mkline); return true },
func(mkline *MkLine) {})
}
// ForEachEnd calls the action for each line, until the action returns false.
// It keeps track of the indentation and all conditional variables.
// At the end, atEnd is called with the last line as its argument.
func (mklines *MkLines) ForEachEnd(action func(mkline *MkLine) bool, atEnd func(lastMkline *MkLine)) bool {
// XXX: To avoid looping over the lines multiple times, it would
// be nice to have an interface LinesChecker that checks a single topic.
// Multiple of these line checkers could be run in parallel, so that
// the diagnostics appear in the correct order, from top to bottom.
// ForEachEnd must not be called within itself.
assert(mklines.indentation == nil)
mklines.indentation = NewIndentation()
mklines.Tools.SeenPrefs = false
result := true
for _, mkline := range mklines.mklines {
mklines.indentation.TrackBefore(mkline)
if !action(mkline) {
result = false
break
}
mklines.indentation.TrackAfter(mkline)
}
if len(mklines.mklines) > 0 {
atEnd(mklines.mklines[len(mklines.mklines)-1])
}
mklines.indentation = nil
return result
}
// defineVar marks a variable as defined in both the current package and the current file.
func (mklines *MkLines) defineVar(mkline *MkLine, varname string) {
mklines.allVars.Define(varname, mkline)
if mklines.extraScope != nil {
mklines.extraScope.Define(varname, mkline)
}
}
func (mklines *MkLines) collectPlistVars() {
// TODO: The PLIST_VARS code above looks very similar.
for _, mkline := range mklines.mklines {
if mkline.IsVarassign() {
switch mkline.Varcanon() {
case "PLIST_VARS":
for _, id := range mkline.ValueFields(resolveVariableRefs(mkline.Value(), mklines, nil)) {
if containsVarUse(id) {
mklines.plistVarSkip = true
} else {
mklines.plistVarAdded[id] = mkline
}
}
case "PLIST.*":
id := mkline.Varparam()
if containsVarUse(id) {
mklines.plistVarSkip = true
} else {
mklines.plistVarSet[id] = mkline
}
}
}
}
}
func (mklines *MkLines) checkAll() {
// checkAll must only be called once, even during tests, since it
// doesn't clean up all its effects on mklines.
assert(mklines.once.FirstTime("checkAll"))
allowedTargets := map[string]bool{
"pre-fetch": true, "do-fetch": true, "post-fetch": true,
"pre-extract": true, "do-extract": true, "post-extract": true,
"pre-patch": true, "do-patch": true, "post-patch": true,
"pre-tools": true, "do-tools": true, "post-tools": true,
"pre-wrapper": true, "do-wrapper": true, "post-wrapper": true,
"pre-configure": true, "do-configure": true, "post-configure": true,
"pre-build": true, "do-build": true, "post-build": true,
"pre-test": true, "do-test": true, "post-test": true,
"pre-install": true, "do-install": true, "post-install": true,
"pre-package": true, "do-package": true, "post-package": true,
"pre-clean": true, "do-clean": true, "post-clean": true}
mklines.lines.CheckCvsID(0, `#[\t ]+`, "# ")
substContext := NewSubstContext(mklines.pkg)
var varalign VaralignBlock
vargroupsChecker := NewVargroupsChecker(mklines)
isHacksMk := mklines.lines.BaseName == "hacks.mk"
if trace.Tracing {
trace.Stepf("Starting main checking loop")
}
mklines.ForEachEnd(
func(mkline *MkLine) bool {
if isHacksMk {
// Needs to be set here because it is reset in MkLines.ForEach.
mklines.Tools.SeenPrefs = true
}
mklines.checkLine(mkline, vargroupsChecker, &varalign, substContext, allowedTargets)
return true
},
func(mkline *MkLine) {
// This check is not done by ForEach because ForEach only
// manages the iteration, not the actual checks.
mklines.indentation.CheckFinish(mklines.lines.Filename)
vargroupsChecker.Finish()
})
substContext.Finish(mklines.EOFLine())
varalign.Finish()
CheckLinesTrailingEmptyLines(mklines.lines)
}
func (mklines *MkLines) checkLine(
mkline *MkLine,
vargroupsChecker *VargroupsChecker,
varalign *VaralignBlock,
substContext *SubstContext,
allowedTargets map[string]bool) {
ck := MkLineChecker{mklines, mkline}
ck.Check()
vargroupsChecker.Check(mkline)
varalign.Process(mkline)
mklines.Tools.ParseToolLine(mklines, mkline, false, false)
substContext.Process(mkline)
switch {
case mkline.IsVarassign():
mklines.checkAllData.target = ""
mkline.Tokenize(mkline.Value(), true) // Just for the side-effect of the warnings.
mklines.checkVarassignPlist(mkline)
varname := mkline.Varname()
mklines.checkAllData.vars.Define(varname, mkline)
case mkline.IsInclude():
mklines.checkAllData.target = ""
if mklines.pkg != nil {
mklines.pkg.checkIncludeConditionally(mkline, mklines.indentation)
}
case mkline.IsDirective():
ck.checkDirective(mklines.checkAllData.forVars, mklines.indentation)
case mkline.IsDependency():
ck.checkDependencyRule(allowedTargets)
mklines.checkAllData.target = mkline.Targets()
case mkline.IsShellCommand():
mkline.Tokenize(mkline.ShellCommand(), true) // Just for the side-effect of the warnings.
}
if mklines.checkAllData.postLine != nil {
mklines.checkAllData.postLine(mkline)
}
}
func (mklines *MkLines) checkVarassignPlist(mkline *MkLine) {
switch mkline.Varcanon() {
case "PLIST_VARS":
for _, id := range mkline.ValueFields(resolveVariableRefs(mkline.Value(), mklines, nil)) {
if !mklines.plistVarSkip && mklines.plistVarSet[id] == nil {
mkline.Warnf("%q is added to PLIST_VARS, but PLIST.%s is not defined in this file.", id, id)
}
}
case "PLIST.*":
id := mkline.Varparam()
if !mklines.plistVarSkip && mklines.plistVarAdded[id] == nil {
mkline.Warnf("PLIST.%s is defined, but %q is not added to PLIST_VARS in this file.", id, id)
}
}
}
// CheckUsedBy checks that this file (a Makefile.common) has the given
// relativeName in one of the "# used by" comments at the beginning of the file.
func (mklines *MkLines) CheckUsedBy(relativeName PkgsrcPath) {
lines := mklines.lines
if lines.Len() < 3 {
return
}
paras := mklines.SplitToParagraphs()
expected := "# used by " + relativeName.String()
found := false
var usedParas []*Paragraph
determineUsedParas := func() {
for _, para := range paras {
var hasUsedBy bool
var hasOther bool
var conflict *MkLine
para.ForEach(func(mkline *MkLine) {
if ok, _ := mkline.IsCvsID(`#[\t ]+`); ok {
return
}
if hasPrefix(mkline.Text, "# used by ") && len(strings.Fields(mkline.Text)) == 4 {
if mkline.Text == expected {
found = true
}
hasUsedBy = true
if hasOther && conflict == nil {
conflict = mkline
}
} else {
hasOther = true
if hasUsedBy && conflict == nil {
conflict = mkline
}
}
})
if conflict != nil {
conflict.Warnf("The \"used by\" lines should be in a separate paragraph.")
} else if hasUsedBy {
usedParas = append(usedParas, para)
}
}
}
determineUsedParas()
if len(usedParas) > 1 {
usedParas[1].FirstLine().Warnf("There should only be a single \"used by\" paragraph per file.")
}
var prevLine *MkLine
if len(usedParas) > 0 {
prevLine = usedParas[0].LastLine()
} else {
prevLine = paras[0].LastLine()
if paras[0].to > 1 {
fix := prevLine.Autofix()
fix.Notef(SilentAutofixFormat)
fix.InsertBelow("")
fix.Apply()
}
}
// TODO: Sort the comments.
// TODO: Discuss whether these comments are actually helpful.
// TODO: Remove lines that don't apply anymore.
if !found {
fix := prevLine.Autofix()
fix.Warnf("Please add a line %q here.", expected)
fix.Explain(
"Since Makefile.common files usually don't have any comments and",
"therefore not a clearly defined purpose, they should at least",
"contain references to all files that include them, so that it is",
"easier to see what effects future changes may have.",
"",
"If there are more than five packages that use a Makefile.common,",
"that file should have a clearly defined and documented purpose,",
"and the filename should reflect that purpose.",
"Typical names are module.mk, plugin.mk or version.mk.")
fix.InsertBelow(expected)
fix.Apply()
}
SaveAutofixChanges(lines)
}
func (mklines *MkLines) SplitToParagraphs() []*Paragraph {
var paras []*Paragraph
lines := mklines.mklines
isEmpty := func(i int) bool {
if lines[i].IsEmpty() {
return true
}
return lines[i].IsComment() &&
lines[i].Text == "#" &&
(i == 0 || lines[i-1].IsComment()) &&
(i == len(lines)-1 || lines[i+1].IsComment())
}
i := 0
for i < len(lines) {
from := i
for from < len(lines) && isEmpty(from) {
from++
}
to := from
for to < len(lines) && !isEmpty(to) {
to++
}
if from != to {
paras = append(paras, NewParagraph(mklines, from, to))
}
i = to
}
return paras
}
// ExpandLoopVar searches the surrounding .for loops for the given
// variable and returns a slice containing all its values, fully
// expanded.
//
// It can only be used during an active ForEach call.
func (mklines *MkLines) ExpandLoopVar(varname string) []string {
// From the inner loop to the outer loop, just in case
// that two loops should ever use the same variable.
for i := len(mklines.indentation.levels) - 1; i >= 0; i-- {
ind := mklines.indentation.levels[i]
mkline := ind.mkline
if mkline.Directive() != "for" {
continue
}
// TODO: If needed, add support for multi-variable .for loops.
resolved := resolveVariableRefs(mkline.Args(), mklines, nil)
words := mkline.ValueFields(resolved)
if len(words) >= 3 && words[0] == varname && words[1] == "in" {
return words[2:]
}
}
return nil
}
// IsUnreachable determines whether the given line is unreachable because a
// condition on the way to that line is not satisfied.
// If unsure, returns false.
//
// Only the current package and Makefile fragment are taken into account.
// The line might still be reachable by another pkgsrc package.
func (mklines *MkLines) IsUnreachable(mkline *MkLine) bool {
// To make this code as simple as possible, the code should operate
// on a high-level AST, where the nodes are If, For and BasicBlock.
//
// See lang/ghc*/bootstrap.mk for good examples how pkglint should
// treat variable assignments. It's getting complicated.
return false
}
func (mklines *MkLines) SaveAutofixChanges() {
mklines.lines.SaveAutofixChanges()
}
func (mklines *MkLines) EOFLine() *MkLine {
return NewMkLineParser().Parse(mklines.lines.EOFLine())
}
// Whole returns a virtual line that can be used for issuing diagnostics
// and explanations, but not for text replacements.
func (mklines *MkLines) Whole() *Line { return mklines.lines.Whole() }