- Added a new top level file that saves the REPORT_BASEDIR from the first
call to the mk/bulk/build script. This is to allow restarting the bulk
build without getting confused by the various timestamps that occur
during a bulk build. It also fixes some problems where the text/plain
report had ended up in a directory different from the other files.
- Changed the default filename for the text/plain report from
"pkgsrc-results-${BUILDDATE}.txt" to simply "report.txt", as the
${BUILDDATE} variable didn't give a hint to when the bulk build had been
started, which is often more important than when the bulk build stopped.
- Changed the way to specify the directory and the URL where the bulk build
reports are made available. See
http://mail-index.netbsd.org/tech-pkg/2005/12/05/0002.html
for details. Specifying the directories using the FTPHOST and FTPURL
variables will be possible for some months, after which it support for
it will be dropped.
This commit is contained in:
rillig
parent
8358f4d35e
commit
1278e33918
5 changed files with 124 additions and 47 deletions
|
|
@ -1,4 +1,4 @@
|
|||
# $NetBSD: bsd.bulk-pkg.mk,v 1.111 2006/01/17 22:35:13 jdc Exp $
|
||||
# $NetBSD: bsd.bulk-pkg.mk,v 1.112 2006/01/21 11:29:31 rillig Exp $
|
||||
|
||||
#
|
||||
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
|
||||
|
|
@ -145,6 +145,10 @@ ORDERFILE?= ${BULKFILESDIR}/.order${BULK_ID}
|
|||
# eventually for looking for leftover files (files not properly deinstalled)
|
||||
STARTFILE?= ${BULKFILESDIR}/.start${BULK_ID}
|
||||
|
||||
# This top level file saves the bulk build ID. As it most often has a time
|
||||
# stamp, it is only generated once and later retrieved from this file.
|
||||
BULK_BUILD_ID_FILE?= ${BULKFILESDIR}/.bulk_build_id
|
||||
|
||||
# File created and used by lintpkgsrc(8) to cache package metadata for
|
||||
# pruning and bulk-upload exclusions.
|
||||
LINTPKGSRC_DB?= ${BULKFILESDIR}/.lintpkgsrc.db${BULK_ID}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/bin/sh
|
||||
# $NetBSD: build,v 1.91 2005/12/05 10:19:14 rillig Exp $
|
||||
# $NetBSD: build,v 1.92 2006/01/21 11:29:32 rillig Exp $
|
||||
|
||||
#
|
||||
# Copyright (c) 1999, 2000 Hubert Feyrer <hubertf@NetBSD.org>
|
||||
|
|
@ -48,8 +48,7 @@ The following options are supported:
|
|||
one.
|
||||
|
||||
-e | --no-email
|
||||
Don't send email when the bulk build is finished, it will put
|
||||
the results into a file (FTP/pkgsrc-results.txt).
|
||||
Don't send email when the bulk build is finished.
|
||||
|
||||
-h | --help
|
||||
Displays this message.
|
||||
|
|
@ -279,6 +278,7 @@ load_pkgsrc_var INDEXFILE
|
|||
load_pkgsrc_var ORDERFILE
|
||||
load_pkgsrc_var STARTFILE
|
||||
load_pkgsrc_var SUPPORTSFILE
|
||||
load_pkgsrc_var BULK_BUILD_ID_FILE
|
||||
|
||||
load_pkgsrc_var BUILDLOG
|
||||
load_pkgsrc_var BROKENFILE
|
||||
|
|
@ -306,6 +306,15 @@ if [ "${BULKFILESDIR}" != "${pkgsrc_dir}" ]; then
|
|||
${MKDIR} "${BULKFILESDIR}"
|
||||
fi
|
||||
|
||||
#
|
||||
# Save the bulk build ID in a file, as it most often contains a time
|
||||
# stamp.
|
||||
#
|
||||
case $restart in
|
||||
no) echo "${REPORT_BASEDIR}" > "${BULK_BUILD_ID_FILE}" \
|
||||
|| die "Could not save the bulk build ID in ${BULK_BUILD_ID_FILE}.";;
|
||||
esac
|
||||
|
||||
#
|
||||
# Install prerequisite packages.
|
||||
#
|
||||
|
|
@ -417,18 +426,22 @@ done
|
|||
# Generate the post-build report.
|
||||
#
|
||||
echo "build> Generating the bulk build report..."
|
||||
BUILDDATE=`date +%Y-%m-%d`
|
||||
mkdir -p "${FTP}"
|
||||
|
||||
bulk_build_id=`cat "${BULK_BUILD_ID_FILE}"` \
|
||||
|| die "Could not read the bulk build ID from ${BULK_BUILD_ID_FILE}."
|
||||
|
||||
report_dir="${REPORTS_DIR}/${bulk_build_id}"
|
||||
${MKDIR} "${report_dir}"
|
||||
( cd "${pkgsrc_dir}" \
|
||||
&& ${PERL5} mk/bulk/post-build \
|
||||
> ${FTP}/pkgsrc-results-${BUILDDATE}.txt
|
||||
> "${report_dir}/${REPORT_TXT_FILE}"
|
||||
) || die "Could not write the results file."
|
||||
|
||||
#
|
||||
# Notify the ADMIN of the finished build.
|
||||
#
|
||||
case $noemail in
|
||||
no) cat "${FTP}/pkgsrc-results-${BUILDDATE}.txt" \
|
||||
no) cat "${report_dir}/${REPORT_TXT_FILE}" \
|
||||
| ${MAIL_CMD} -s "pkgsrc ${OPSYS} ${OS_VERSION}/${MACHINE_ARCH} bulk build results $BUILDDATE" "$ADMIN"
|
||||
esac
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# $NetBSD: build.conf-example,v 1.32 2005/12/03 01:00:37 rillig Exp $
|
||||
# $NetBSD: build.conf-example,v 1.33 2006/01/21 11:29:32 rillig Exp $
|
||||
#
|
||||
# This is an example configuration file for pkgsrc bulk builds.
|
||||
# Actually it's a shell script that is sourced in by the pre-build,
|
||||
|
|
@ -61,11 +61,22 @@ ADMIN="you@some.whe.re"
|
|||
# Who the report is signed by
|
||||
ADMINSIG="-Your Name"
|
||||
|
||||
# Some paths for output files and paths to log files
|
||||
FTPx=`date +%Y%m%d.%H%M`
|
||||
FTPURL="pub/NetBSD/pkgstat/${FTPx}" # relative to ~ftp !
|
||||
FTP="/home/ftp/${FTPURL}" # absolute base path
|
||||
FTPHOST="ftp://ftp.machi.ne" # host for broken.html
|
||||
# The directory where the final reports are collected.
|
||||
#REPORTS_DIR="$HOME/bulk-logs"
|
||||
REPORTS_DIR="/home/ftp/pub/NetBSD/pkgstat"
|
||||
|
||||
# The URL where the final reports will be available.
|
||||
REPORTS_URL="ftp://localhost/pub/NetBSD/pkgstat"
|
||||
|
||||
# The basename of the directory of the current bulk build. This
|
||||
# directory will be created under ${REPORTS_DIR}. After uploading the
|
||||
# report, it will also be available under ${REPORTS_URL}.
|
||||
REPORT_BASEDIR=`date +%Y%m%d.%H%M`
|
||||
|
||||
# The basenames of the the two report files, which will both be created
|
||||
# in ${REPORTS_DIR}/${REPORT_BASEDIR}.
|
||||
REPORT_HTML_FILE="report.html"
|
||||
REPORT_TXT_FILE="report.txt"
|
||||
|
||||
#
|
||||
# Uploading binary packages
|
||||
|
|
@ -88,10 +99,3 @@ MKSUMS=yes
|
|||
# Destination for packages and rsync options
|
||||
RSYNC_DST=ftp.NetBSD.org:/pub/NetBSD/packages/pkgsrc-200xQy/NetBSD-a.b.c/i386
|
||||
RSYNC_OPTS='-e ssh'
|
||||
|
||||
###########################################################################
|
||||
### No changes should be needed below this line !!!
|
||||
###########################################################################
|
||||
|
||||
# Name of the (generated) HTML file which lists all broken pkgs
|
||||
REPORT="$FTP/broken.html"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/pkg/bin/perl
|
||||
# $NetBSD: post-build,v 1.62 2005/12/03 13:39:04 rillig Exp $
|
||||
# $NetBSD: post-build,v 1.63 2006/01/21 11:29:32 rillig Exp $
|
||||
#
|
||||
# Collect stuff after a pkg bulk build
|
||||
#
|
||||
|
|
@ -41,6 +41,23 @@ sub my_system (@) {
|
|||
return system(@_);
|
||||
}
|
||||
|
||||
sub readfirstline($) {
|
||||
my ($fname) = @_;
|
||||
my ($contents);
|
||||
|
||||
open(F, "<", $fname)
|
||||
or pb_die($fname, "Cannot be read: $!");
|
||||
|
||||
defined($contents = <F>)
|
||||
or pb_die($fname, "Must not be empty.");
|
||||
chomp($contents);
|
||||
|
||||
close(F)
|
||||
or pb_die($fname, "Cannot be closed: $!");
|
||||
|
||||
return $contents;
|
||||
}
|
||||
|
||||
#
|
||||
# Load configuration variables from the bulk.conf file, which is a shell
|
||||
# script.
|
||||
|
|
@ -59,7 +76,16 @@ sub get_build_conf_vars(@) {
|
|||
my ($is_set, $value);
|
||||
|
||||
foreach my $varname (@varnames) {
|
||||
open(CMD, ". '$BULK_BUILD_CONF'; echo \"\${${varname}+set}\"; echo \"\${${varname}-}\" |")
|
||||
my $cmd = join("\n", (
|
||||
#"set -eu", # TODO: Should be enabled soon
|
||||
". '${BULK_BUILD_CONF}'",
|
||||
". mk/bulk/post-build-conf",
|
||||
"check_config_vars",
|
||||
"echo \"\${${varname}+set}\"",
|
||||
"echo \"\${${varname}-}\""
|
||||
));
|
||||
|
||||
open(CMD, "${cmd} |")
|
||||
or pb_die($BULK_BUILD_CONF, "Could not evaluate configuration file.");
|
||||
|
||||
chomp($is_set = <CMD>);
|
||||
|
|
@ -91,15 +117,17 @@ sub get_build_conf_vars(@) {
|
|||
|
||||
get_build_conf_vars(
|
||||
'ADMINSIG', # "-Your Name"
|
||||
'FTPURL', # "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
|
||||
'FTP', # "/disk1/ftp/${FTPURL}"
|
||||
'FTPHOST', # ftp://ftp.machi.ne/
|
||||
'REPORT', # "broken.html"
|
||||
'REPORTS_URL', # "ftp://ftp.example.com/pub/pkgsrc/misc/pkgstat"
|
||||
'REPORTS_DIR', # "$HOME/bulk-logs"
|
||||
# REPORT_BASEDIR often depends on a timestamp, which has been saved at
|
||||
# the beginning of the bulk build in the BULK_BUILD_ID_FILE. It will be
|
||||
# retrieved later.
|
||||
'REPORT_HTML_FILE', # "report.html"
|
||||
'REPORT_TXT_FILE', # "report.txt"
|
||||
'USR_PKGSRC', # "/usr/pkgsrc"
|
||||
'osrev', # `uname -r`
|
||||
);
|
||||
|
||||
my $reportf = basename($vars{REPORT});
|
||||
|
||||
my $os = `uname -s`;
|
||||
chomp $os;
|
||||
|
|
@ -151,6 +179,7 @@ get_mk_conf_vars(qw(
|
|||
SUPPORTSFILE
|
||||
X11BASE
|
||||
PKG_TOOLS_BIN
|
||||
BULK_BUILD_ID_FILE
|
||||
));
|
||||
|
||||
my $bulk_dbfile_base = basename($vars{BULK_DBFILE});
|
||||
|
|
@ -160,6 +189,17 @@ my $supportsfile_base = basename($vars{SUPPORTSFILE});
|
|||
my $indexfile_base = basename($vars{INDEXFILE});
|
||||
my $orderfile_base = basename($vars{ORDERFILE});
|
||||
|
||||
my $reports_url = $vars{"REPORTS_URL"};
|
||||
my $reports_dir = $vars{"REPORTS_DIR"};
|
||||
my $report_basedir = readfirstline($vars{"BULK_BUILD_ID_FILE"});
|
||||
my $report_html_file = $vars{"REPORT_HTML_FILE"};
|
||||
my $report_txt_file = $vars{"REPORT_TXT_FILE"};
|
||||
|
||||
my $report_url = "${reports_url}/${report_basedir}";
|
||||
my $report_dir = "${reports_dir}/${report_basedir}";
|
||||
my $report_html_path = "${report_dir}/${report_html_file}";
|
||||
my $report_txt_path = "${report_dir}/${report_txt_file}";
|
||||
|
||||
my $startdate = (stat($vars{STARTFILE}))[9];
|
||||
my $enddate = '';
|
||||
if (!defined($startdate) || $startdate == 0) {
|
||||
|
|
@ -187,31 +227,27 @@ sub print_report_header() {
|
|||
print("--------------------------------------------------------------\n");
|
||||
}
|
||||
|
||||
my_system("mkdir", "-p", "--", $vars{FTP});
|
||||
my_system("mkdir", "-p", "--", $report_dir);
|
||||
|
||||
# Copy over the output from the build process
|
||||
chdir($vars{"BULKFILESDIR"}) or pb_die($vars{"BULKFILESDIR"}, "Cannot change directory.");
|
||||
my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X $vars{FTP}");
|
||||
my_system("find . -name $vars{BROKENFILE} -print -o -name $vars{BROKENWRKLOG} -print | $vars{PAX} -r -w -X ${report_dir}");
|
||||
|
||||
# Copy over the cache files used during the build
|
||||
foreach my $f qw(BULK_DBFILE DEPENDSTREEFILE DEPENDSFILE SUPPORTSFILE INDEXFILE ORDERFILE) {
|
||||
if (-f $vars{$f}) {
|
||||
my_system("cp", "--", $vars{$f}, $vars{FTP});
|
||||
my_system("cp", "--", $vars{$f}, $report_dir);
|
||||
}
|
||||
}
|
||||
|
||||
chdir($vars{FTP}) or pb_die($vars{"FTP"}, "Cannot change directory.");
|
||||
chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
|
||||
writeReport();
|
||||
|
||||
#
|
||||
# Adjust "last" symlink
|
||||
#
|
||||
{
|
||||
my ($base, $dir) = ($vars{FTP} =~ m|^(.*)/([^/]*)$|);
|
||||
|
||||
unlink("$base/last");
|
||||
symlink($dir, "$base/last");
|
||||
}
|
||||
unlink("${reports_dir}/last");
|
||||
symlink($report_basedir, "${reports_dir}/last");
|
||||
|
||||
#
|
||||
# Generate leftovers-$vars{MACHINE_ARCH}.html: files not deleted
|
||||
|
|
@ -219,7 +255,7 @@ writeReport();
|
|||
# and linked from leftovers-$vars{MACHINE_ARCH}.html
|
||||
#
|
||||
{
|
||||
chdir($vars{FTP});
|
||||
chdir($report_dir) or pb_die($report_dir, "Cannot change directory.");
|
||||
my_system("mkdir", "-p", "leftovers-$vars{MACHINE_ARCH}");
|
||||
|
||||
# Find files since last build:
|
||||
|
|
@ -270,7 +306,7 @@ writeReport();
|
|||
EOOUT
|
||||
foreach (@leftovers) {
|
||||
chomp;
|
||||
print OUT "<a href=\"$vars{FTPHOST}/$vars{FTPURL}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
|
||||
print OUT "<a href=\"${report_url}/leftovers-$vars{MACHINE_ARCH}$_\">$_</a>\n";
|
||||
}
|
||||
print OUT <<EOOUT2;
|
||||
</pre>
|
||||
|
|
@ -369,11 +405,11 @@ EOF
|
|||
Packages not listed here resulted in a binary package. The build
|
||||
report, including logs of failed/not-packaged is available from:
|
||||
|
||||
$vars{FTPHOST}/$vars{FTPURL}/$reportf
|
||||
${report_url}/${report_html_file}
|
||||
EOF
|
||||
}
|
||||
|
||||
open(HTML, ">$vars{REPORT}") or die "Can't write $vars{REPORT}: $!\n";
|
||||
open(HTML, ">", $report_html_path) or die "Can't write ${report_html_path}: $!";
|
||||
print HTML <<EOHTML;
|
||||
<html>
|
||||
<head>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# $NetBSD: post-build-conf,v 1.10 2006/01/07 22:27:56 rillig Exp $
|
||||
# $NetBSD: post-build-conf,v 1.11 2006/01/21 11:29:33 rillig Exp $
|
||||
#
|
||||
|
||||
# This file is included after the build.conf file by the "build" and
|
||||
|
|
@ -39,7 +39,9 @@ show_config_vars() {
|
|||
pbc_section "Building the packages" \
|
||||
PKGLIST NICE_LEVEL
|
||||
pbc_section "Generating the report" \
|
||||
ADMIN ADMINSIG FTPHOST FTP
|
||||
ADMIN ADMINSIG \
|
||||
REPORTS_URL REPORTS_DIR \
|
||||
REPORT_BASEDIR REPORT_HTML_FILE REPORT_TXT_FILE
|
||||
pbc_section "Uploading binary packages" \
|
||||
UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS \
|
||||
RSYNC_DST RSYNC_OPTS
|
||||
|
|
@ -51,7 +53,7 @@ export_config_vars() {
|
|||
export PRUNEDISTFILES ftp_proxy http_proxy
|
||||
export PKGLIST NICE_LEVEL ADMIN ADMINSIG
|
||||
export UPDATE_VULNERABILITY_LIST PRUNEPACKAGES MKSUMS SIGN_AS
|
||||
export RSYNC_DST RSYNC_OPTS FTPHOST FTP
|
||||
export RSYNC_DST RSYNC_OPTS
|
||||
case ${MAKECONF+set} in
|
||||
"set") export MAKECONF;;
|
||||
esac
|
||||
|
|
@ -154,6 +156,20 @@ pbc_checkdefined() {
|
|||
|
||||
# usage: check_config_vars
|
||||
check_config_vars() {
|
||||
|
||||
# Handle legacy variables FTP, FTPHOST, FTPURL, REPORT.
|
||||
# Can be removed after 2006Q2.
|
||||
case ${FTP+set},${REPORTS_DIR+set} in
|
||||
set,set)
|
||||
pbc_die "FTP and REPORTS_DIR must not be set both.";;
|
||||
set,)
|
||||
REPORTS_URL="${FTPHOST}"`dirname "${FTPURL}"`
|
||||
REPORTS_DIR=`dirname "$FTP"`
|
||||
REPORT_BASEDIR=`basename "${FTP}"`
|
||||
REPORT_HTML_FILE=`basename "${REPORT}"`
|
||||
REPORT_TXT_FILE="report.txt";;
|
||||
esac
|
||||
|
||||
# section "System information"
|
||||
pbc_checknonempty osrev
|
||||
pbc_checknonempty arch
|
||||
|
|
@ -184,8 +200,12 @@ check_config_vars() {
|
|||
# section "Generating the report"
|
||||
# no checks for ADMIN
|
||||
# no checks for ADMINSIG
|
||||
# no checks for FTPHOST
|
||||
# no checks for FTP
|
||||
pbc_checkurl REPORTS_URL
|
||||
pbc_checkpathname REPORTS_DIR
|
||||
pbc_checkbasename REPORT_BASEDIR
|
||||
pbc_checkbasename REPORT_HTML_FILE
|
||||
pbc_checkbasename REPORT_TXT_FILE
|
||||
|
||||
|
||||
# section "Uploading binary packages"
|
||||
pbc_checkyesno UPDATE_VULNERABILITY_LIST
|
||||
|
|
|
|||
Loading…
Reference in a new issue