pkgsrc/mk/bulk/post-build

#!/usr/pkg/bin/perl
# $NetBSD: post-build,v 1.49 2005/01/13 19:29:28 tv Exp $
#
# Collect stuff after a pkg bulk build
#
# (c) 2000 Hubert Feyrer, All Rights Reserved.
#

use File::Basename;
use POSIX qw(strftime);
use strict;
use warnings;

my %vars;
my $verbose = 1; # set to 2 to see more command execution detail

sub my_system (@) {
	print STDERR '> '.join(' ', @_)."\n" if ($verbose >= 2);
	system(@_);
}

# Where config vars are stored (/bin/sh syntax)

my $BULK_BUILD_CONF = $ENV{BULK_BUILD_CONF} || (dirname($0).'/build.conf');
$BULK_BUILD_CONF = "./$BULK_BUILD_CONF" if ($BULK_BUILD_CONF !~ m:^/:);

# Dig given variable out of config file, and set it
sub getconf (@) {
	open(I, ". $BULK_BUILD_CONF; for var in ".join(' ', @_)."; do eval echo \\\${\$var}; done |") || die 'cannot open pipe';
	
	foreach my $var (@_) {
		$vars{$var} = <I>;
		chomp $vars{$var};
		die "\$$var not defined by $BULK_BUILD_CONF" if ($vars{$var} eq '');

		print STDERR "> $var=$vars{$var}\n" if ($verbose >= 2);
	}
}

getconf(
	'ADMINSIG',		# "-Your Name"
	'FTPURL',		# "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
	'FTP',			# "/disk1/ftp/${FTPURL}"
	'FTPHOST',		# ftp://ftp.machi.ne/
	'REPORT',		# "broken.html"
	'USR_PKGSRC',		# "/usr/pkgsrc"
	'arch', 		# `uname -m`
	'osrev',	       	# `uname -r`
);

my $reportf = basename($vars{REPORT});

my $os = `uname -s`;
chomp $os;

my $BMAKE = $ENV{BMAKE} || die '$BMAKE not defined in environment';

sub getmakevars (@) {
	open(I, "cd $vars{USR_PKGSRC}/pkgtools/pkglint && $BMAKE show-vars BATCH=1 VARNAMES='".join(' ', @_)."' |") || die 'cannot open pipe';

	foreach my $var (@_) {
		$vars{$var} = <I>;
		chomp $vars{$var};
		die "\${$var} not defined by $BMAKE" if ($vars{$var} eq '');

		print STDERR "> $var=$vars{$var}\n" if ($verbose >= 2);
	}
}

# Extract the names of the files used for the build log and broken build logs.
# These have defaults set by bsd.bulk-pkg.mk and may be overridden in
# /etc/mk.conf
getmakevars(qw(
	BROKENFILE
	BROKENWRKLOG
	BULK_DBFILE
	DEPENDSFILE
	DEPENDSTREEFILE
	FIND
	GTAR
	INDEXFILE
	LOCALBASE
	ORDERFILE
	PKG_DBDIR
	STARTFILE
	SUPPORTSFILE
	X11BASE
));

my $bulkdbfile = basename($vars{BULK_DBFILE});
my $dtfile = basename($vars{DEPENDSTREEFILE});
my $depfile = basename($vars{DEPENDSFILE});
my $supfile = basename($vars{SUPPORTSFILE});
my $indfile = basename($vars{INDEXFILE});
my $ordfile = basename($vars{ORDERFILE});

my $startdate = (stat($vars{STARTFILE}))[9];
my $enddate = '';
if ($startdate == 0) {
	$startdate = "unknown";
} else {
	local $ENV{TZ} = "UTC";
	$startdate = strftime("%c %Z", gmtime($startdate));
	$enddate = strftime("%c %Z", gmtime(time()));
}

my $report_head = <<EOF;
Package                    Breaks  Maintainer
--------------------------------------------------------------
EOF
my $report_form = <<EOF;
@<<<<<<<<<<<<<<<<<<<<<<<<< @<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<
EOF
# $pkg, $nbrokenby, $maintainer


my_system("mkdir -p $vars{FTP}");

# Copy over the output from the build process
chdir($vars{USR_PKGSRC});
my_system("/bin/ls -1 $vars{BROKENFILE} $vars{BROKENWRKLOG} */*/$vars{BROKENFILE} */*/$vars{BROKENWRKLOG} 2>/dev/null | $vars{GTAR} -T - -plcf - | (cd $vars{FTP}; $vars{GTAR} -plxf -)");

# Copy over the cache files used during the build
foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) {
	my_system("cp $vars{$f} $vars{FTP}") if -f $vars{$f};
}

chdir($vars{FTP});
writeReport();

#
# Adjust "last" symlink
#
{
	my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|);

	unlink("$base/last");
	symlink($dir, "$base/last");
}

#
# Generate leftovers-$vars{arch}.html: files not deleted
# Leftover files are copied to leftovers-$vars{arch} dir, and linked from
# leftovers-$vars{arch}.html
#
{
	chdir($vars{FTP});
	my_system("mkdir -p leftovers-$vars{arch}");

	# Find files since last build:
	my $leftovers_txt = "leftovers-$vars{arch}.txt";
	my $leftovers_html = "leftovers-$vars{arch}.html";

	my_system("$vars{FIND} $vars{LOCALBASE}/ -newer $vars{STARTFILE} -type f -print >>$leftovers_txt");
	my_system("$vars{FIND} $vars{X11BASE}/ -newer $vars{STARTFILE} -type f -print >>$leftovers_txt");

	# Strip perl-files:
	my $perlfiles;
	{
		local $/;
		undef $/;
		$perlfiles = `pkg_info -qL perl*`;
	}

	my $perlfiles_pattern = $perlfiles;
	$perlfiles_pattern =~ s/\n/|/g;
	$perlfiles_pattern =~ s/|$//;

	open (LEFT, $leftovers_txt) or die "can't read $leftovers_txt: $!";
	my @left = <LEFT>;
	close (LEFT);
	my @leftovers = grep(!/^(?:${perlfiles_pattern})$/, @left);

	if (index($vars{PKG_DBDIR}, $vars{LOCALBASE}) == 0) {
		# If PKG_DBDIR is inside LOCALBASE, exclude it from the leftovers.
		@leftovers = grep { index($_, $vars{PKG_DBDIR}) != 0 } @leftovers;
	}

	open (LEFT, ">$leftovers_txt") or die "can't write $leftovers_txt: $!";
	print LEFT @leftovers;
	close (LEFT);

	if (scalar(@leftovers)) {
		# Store leftovers, for easier identification:
		my_system("$vars{GTAR} -T $leftovers_txt -plcf - | (cd leftovers-$vars{arch}; $vars{GTAR} -plxf -)");
	}

	# Add links to leftover list:
	open (OUT, "> $leftovers_html")
		or die "can't write $leftovers_html";
	print OUT <<EOOUT;
<html>
<body>
<pre>
EOOUT
	foreach (@leftovers) {
		chomp;
	    	print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{arch}$_\">$_</a>\n";
	}
	print OUT <<EOOUT2;
</pre>
</body>
</html>
EOOUT2
    	close(OUT);
}

# print the result of a single broken package
sub pkgResult ($$) {
	my ($pinfo, $state) = @_;
	my $pkg = $pinfo->{pkg};
	my $nbrokenby = $pinfo->{nbrokenby};
	my $nerrors = $pinfo->{nerrors};

	my @idents = `$vars{FIND} $vars{USR_PKGSRC}/$pkg -type f -print | xargs grep \\\$NetBSD`;
	my $datetime = "";
	my $file = "";
	my $ver = "";
	foreach my $ident (@idents) {
		$ident =~ /\$[N]etBSD: ([^ ]*),v ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) Exp \$/;
		if ("$3 $4" gt $datetime) {
			$datetime = "$3 $4";
			$file = $1;
			$ver = $1;
		}
	}

	my $maintainer = `grep ^MAINTAINER $vars{USR_PKGSRC}/$pkg/Makefile`;
	$maintainer =~ s/MAINTAINER=[ \t]*//;
	$maintainer =~ s/</&lt;/g;
	$maintainer =~ s/>/&gt;/g;
	chomp($maintainer);

	(my $state_style = $state) =~ s/ //g;

	my $nbrokenby_html = '<td>&nbsp;</td>'; 
	$nbrokenby_html =
		'<td align="right" class="pkg-'.$state_style.'">'.$nbrokenby.'</td>'
		if $nbrokenby > 0;

	if ($pinfo->{nerrors} != 0 && $verbose && $state eq "broken") {
		print swrite($report_form, $pkg, $nbrokenby > 0 ? $nbrokenby : "", $maintainer);
	}

	return <<EOHTML;
<tr>
  <td><a class="pkg-$state_style" href="$pinfo->{bf}" title="build log for $pkg">$pkg</a></td>
  $nbrokenby_html
  <td>$file</td>
  <td>$maintainer</td>
</tr>

EOHTML
}

# write the build report
sub writeReport {
	my $broken = getBroken();
	my $nbroken = scalar(@{$broken->{"broken"}});
	my $nbrokendep = scalar(@{$broken->{"broken depends"}});
	my $nunpackaged = scalar(@{$broken->{"not packaged"}});
	my $nbrokentot = $nbroken + $nbrokendep;
	my $ntotal = $nunpackaged + $nbroken + $nbrokendep;

	# determine the number of packages attempted, and then successful
	open(ORDER, $vars{ORDERFILE}) || die "can't open $vars{ORDERFILE}: $!";
	my @order = <ORDER>;
	close(ORDER);
	my $nattempted = scalar(@order);
	my $nsuccessful = $nattempted - $ntotal;

	if ($verbose) {
		print <<EOF;
pkgsrc bulk build results
$os $vars{osrev}/$vars{arch}

Summary:

EOF
		my $summary_form = <<EOF;
        @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<  @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
EOF
		print swrite($summary_form, "Build started:", $startdate);
		print swrite($summary_form, "Build ended:", $enddate);
		print swrite($summary_form, '', '');
		print swrite($summary_form, "Successfully packaged:", $nsuccessful);
		print swrite($summary_form, "Packages really broken:", $nbroken);
		print swrite($summary_form, "Pkgs broken due to them:", $nbrokendep);
		print swrite($summary_form, "Total broken:", $nbrokentot);
		print swrite($summary_form, "Not packaged:", $nunpackaged);
		print swrite($summary_form, "Total:", $ntotal);
		print <<EOF;

Packages not listed here resulted in a binary package. The build
report, including logs of failed/not-packaged is available from:

$vars{FTPHOST}/$vars{FTPURL}/$reportf
EOF
	}

	open(HTML, ">$vars{REPORT}") or die "Can't write $vars{REPORT}: $!\n";
	print HTML <<EOHTML;
<html>
<head>
<title>$os $vars{osrev}/$vars{arch} bulk package build</title>
<style type="text/css">
<!--

body {
	Font-Family: Tahoma, Verdana, sans-serif;
	Line-Height: 1.3em;
	Text-Decoration: None;
	Color: black;
	Background-Color: white;
	Border-Width: 0;
}

table {
	Border-Width: 0;
}

table td {
	Font-Family: Tahoma, Verdana, sans-serif;
	line-height: 1em;
}

a:link {
	Color: #3535c5;
}

a:visited {
	Color: #700080;
}

a:hover {
	Color: #6565e5;
	Text-Decoration: underline;
}

tr {
	Vertical-Align: top;
}

td {
	Vertical-Align: top;
}

h1 {
	Font-Size: 3.5ex;
	Line-Height: 1em;
	Color: #000066;
}

h2 {
	Font-Size: 2.5ex;
	Line-Height: 1em;
	Color: #660000;
}

h3 {
	Font-Size: 2ex;
	Color: #660066;
}

h4 {
	Font-Size: 1.8ex;
	Color: #006600;
}

tt.filename {
	Line-Height: 1.3em;
	Color: #AA0000;
}

.pkgname {
	Font-Family: Arial, Helvetica, Courier, fixed;
	Font-Style: Italic;
	Text-Decoration: none;
	Line-Height: 1.3em;
}

.pkg-broken {
	Color: red;
}

.pkg-brokendepends {
	Color: orange;
}

.pkg-notpackaged {
	Color: blue;
}
-->
</style>
</head>

<body bgcolor="white" text="black" link="#3535c5" vlink="#700080" alink="#3535c5">

<a name="top"></a>
<h1>pkgsrc bulk build results</h1>
<h2>$os $vars{osrev}/$vars{arch}</h2>

<h3>Summary</h3>

<table>
<tr>
  <td>Build started:			<td align="right">$startdate</td>
</tr>
<tr>
  <td>Build ended:			<td align="right">$enddate</td>
</tr>
<tr>
  <td>&nbsp;</td>			<td>&nbsp;</td>
</tr>
<tr>
  <td>Successfully packaged:</td>	<td align="right">$nsuccessful</td>
</tr>
<tr class="pkg-broken">
  <td>Packages really broken:</td>	<td align="right">$nbroken</td>
</tr>
<tr class="pkg-brokendepends">
  <td>Packages broken due to them:</td>	<td align="right">$nbrokendep</td>
</tr>
<tr>
  <td>Total broken:</td>		<td align="right">$nbrokentot</td>
</tr>
<tr class="pkg-notpackaged">
  <td>Not packaged:</td>		<td align="right">$nunpackaged</td>
</tr>
<tr>
  <td>Total:</td>			<td align="right">$ntotal</td>
</tr>
</table>

<p>
  Packages not listed here resulted in a <a
  href="../../packages/" title="binary packages for $os $vars{osrev}/$vars{arch}">binary
  package</a>. Results of failed packages are available below.
</p>

<p>
  Files leftover from the build (because of broken PLISTs, etc.) can be
  found in <a href="leftovers-$vars{arch}.html" title="leftover files">this
  list</a>.
</p>

<p>
  Jump to:<br/>
  <ul>
    <li><a href="#broken">Broken packages</a></li>
    <li><a href="#broken depends">Broken dependencies</a></li>
    <li><a href="#not packaged">Not packaged</a></li>
  </ul>
</p>

EOHTML

	my %state_head = (
		"broken" => "Broken packages",
		"broken depends" => "Broken dependencies",
		"not packaged" => "Not packaged"
	);

	# only include broken packages in the email, see also pkgResult().
	if ($verbose && scalar(@{$broken->{"broken"}})) {
		print "\n\n$state_head{broken}\n\n";
		print $report_head;
	}

	foreach my $state ("broken", "broken depends", "not packaged") {
		next unless scalar(@{$broken->{$state}});

		print HTML <<EOHTML;

<a name="$state"></a>
<h2>$state_head{$state}</h2>
<table width="100%">
<tr align="left">
  <th width="30%">Package</th>
  <th>Breaks</th>
  <th>File touched last</th>
  <th>Maintainer</th>
</tr>

EOHTML
		foreach my $pinfo (@{$broken->{$state}}) {
			print HTML pkgResult($pinfo, $state);
		}

		print HTML <<EOHTML;
</table>
<hr>
<a href="#top">Up to top</a><br/>
<hr>
EOHTML
	}

	print HTML <<EOHTML;
<hr>
<p>
The following cache files were used during the build:
</p>
<ul>
<li>The <a href="$bulkdbfile">SPECIFIC_PKGS bulk database file</a>.</li>
<li>The <a href="$dtfile">depends tree file</a>.</li>
<li>The <a href="$depfile">depends file</a>.</li>
<li>The <a href="$supfile">supports file</a>.</li>
<li>The <a href="$indfile">index file</a>.</li>
<li>The <a href="$ordfile">build order file</a>.</li>
</ul>
<hr>

<p>    
<ul>
<!-- <li>See the list of <a href="../index.html">all log files</a>. -->
<li>Visit the <a href="http://www.NetBSD.org">NetBSD web site</a>.
<li>Learn more about
    <a href="http://www.NetBSD.org/Documentation/software/packages.html">
    The NetBSD Packages Collection</a>.
</ul>
</p>
</body>
</html>
EOHTML
	close(HTML);

	if ($verbose) {
		print "\n\n$vars{ADMINSIG}\n\n";
		print "[* This message was created by the Packages Collection bulk build software *]\n";
	}
}

# get and sort the broken packages
sub getBroken {
	my $res = {
		'broken' => [],
		'broken depends' => [],
		'not packaged' => [],
	};

	open (BF, $vars{BROKENFILE}) || return $res;
	my @in = <BF>;
	close (BF);

	foreach (@in) {
		chomp;
		my ($nerrors, $bf, $nbrokenby) = split;
		my $pkg = $bf;
		$pkg =~ s,/$vars{BROKENFILE},,;
		my %tmp = (
			bf => $bf,
			pkg => $pkg,
			nbrokenby => $nbrokenby,
			nerrors => $nerrors,
		);

		if ($nerrors > 0) {
			push(@{$res->{"broken"}}, \%tmp);
		} elsif ($nerrors == -1) {
			push(@{$res->{"broken depends"}}, \%tmp);
		} else {
			push(@{$res->{"not packaged"}}, \%tmp);
		}
	}

	# sort pkgs in each state
	foreach my $state ("broken", "broken depends", "not packaged") {
		$res->{$state} = [ sort { $a->{pkg} cmp $b->{pkg} } @{$res->{$state}} ];
	}

	return $res;
}

sub swrite {
	my $format = shift;
	$^A = "";
	formline($format, @_);
	$^A =~ s/\n\n/\n/;
	return $^A;
}