kpriv/pkgsrc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

pbulk-base-0.51: Extend network support for name resolution and IPv6

Browse source 
(where applicable).
This commit is contained in:
joerg 2015-12-07 16:52:39 +00:00
parent 731e32d1b9
commit 0b6285ee53
10 changed files with 212 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkgtools/pbulk-base/Makefile
View file

@ -1,6 +1,6 @@
# $NetBSD: Makefile,v 1.19 2015/11/30 22:38:16 joerg Exp $
# $NetBSD: Makefile,v 1.20 2015/12/07 16:52:40 joerg Exp $
DISTNAME= pbulk-base-0.50
DISTNAME= pbulk-base-0.51
COMMENT= Core components of the modular bulk build framework
.include "../../pkgtools/pbulk/Makefile.common"

196
pkgtools/pbulk/files/pbulk/lib/netaddr.c
View file

@ -1,4 +1,4 @@
/* $NetBSD: netaddr.c,v 1.10 2015/11/30 22:38:16 joerg Exp $ */
/* $NetBSD: netaddr.c,v 1.11 2015/12/07 16:52:39 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -33,48 +33,180 @@
#include <nbcompat.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <nbcompat/netdb.h>
#include <nbcompat/stdlib.h>
#include <nbcompat/string.h>
#include "pbulk.h"
int
parse_sockaddr_in(const char *str, struct sockaddr_in *addr)
struct listener {
LIST_ENTRY(listener) ls_link;
struct event ls_event;
int ls_fd;
};
static LIST_HEAD(, listener) all_listener = LIST_HEAD_INITIALIZER(&all_listener);
void
shutdown_listeners(void)
{
const char *port_sep;
char *port_end;
struct in_addr in;
unsigned long tmp;
if ((port_sep = strrchr(str, ':')) != NULL) {
char *addr_part = strdup(str);
addr_part[port_sep - str] = '\0';
if (inet_aton(addr_part, &in) == 0) {
free(addr_part);
return -1;
}
free(addr_part);
str = port_sep + 1;
} else {
memset(&in, 0, sizeof(in));
struct listener *ls;
while ((ls = LIST_FIRST(&all_listener)) != NULL) {
LIST_REMOVE(ls, ls_link);
event_del(&ls->ls_event);
close(ls->ls_fd);
free(ls);
}
}
errno = 0;
tmp = strtoul(str, &port_end, 10);
if (*str == '\0' || *port_end != '\0' || errno != 0 || tmp > 0xfffful)
static int
split_netaddr(const char *str, char **host, char **port, int *numeric)
{
const char *sep = strrchr(str, ':');
if (sep == NULL) {
*numeric = 0;
*host = NULL;
*port = xstrdup(str);
return 0;
}
if (sep[1] == '\0') {
warnx("invalid network address with empty port: %s", str);
return -1;
memset(addr, 0, sizeof(struct sockaddr_in));
addr->sin_port = htons((in_port_t)tmp);
addr->sin_addr = in;
#if !defined(__sun) && !defined(__hpux) && !defined(__INTERIX) && \
!defined(__digital__) && !defined(__linux) && !defined(__sgi)
addr->sin_len = sizeof(*addr);
#endif
addr->sin_family = AF_INET;
}
if (sep == str) {
/* Consider ":port" as equivalent to just "port". */
*numeric = 0;
*host = NULL;
*port = xstrdup(sep + 1);
return 0;
}
if (str[0] == '[' && sep[-1] == ']') {
/* Recognize URL-style numeric IPv6 addresses in []. */
*numeric = 1;
*host = xstrndup(str + 1, sep - str - 2);
*port = xstrdup(sep + 1);
return 0;
}
if (memchr(str, ':', sep - str) != NULL) {
warnx("invalid network with colon in host name: %s", str);
return -1;
}
*numeric = 0;
*host = xstrndup(str, sep - str);
*port = xstrdup(sep + 1);
return 0;
}
static struct addrinfo *
prepare_getaddrinfo(const char *netaddr, int passive)
{
struct addrinfo hints, *result;
char *host, *port;
int numeric, rv;
if (split_netaddr(netaddr, &host, &port, &numeric))
return NULL;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
if (passive)
hints.ai_flags |= AI_PASSIVE;
if (numeric)
hints.ai_flags |= AI_NUMERICHOST;
rv = getaddrinfo(host, port, &hints, &result);
free(host);
free(port);
if (rv != 0) {
if (rv == EAI_SYSTEM)
warn("getaddrinfo failed");
else
warnx("getaddrinfo failed: %s", gai_strerror(rv));
return NULL;
}
return result;
}
int
connect_sockaddr(const char *netaddr)
{
struct addrinfo *result, *res;
int s;
if ((result = prepare_getaddrinfo(netaddr, 0)) == NULL)
return -1;
s = -1;
for (res = result; res != NULL; res = res->ai_next) {
s = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (s == -1)
continue;
if (connect(s, res->ai_addr, res->ai_addrlen) != -1)
break;
close(s);
s = -1;
}
freeaddrinfo(result);
return s;
}
static int
bind_and_listen(struct addrinfo *res, void (*cb)(int, void *))
{
static const int one = 1;
struct listener *ls;
int s;
s = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (s == -1)
return 0;
if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) {
close(s);
return 0;
}
#ifdef IPV6_V6ONLY
/*
* Disable mapped IPv4, which is still the default at least on Linux.
* It doesn't matter if an error occurs, but success allows binding
* to both IPv4 and IPv6 wild card addresses. Otherwise,
* the second bind would fail.
*/
if (res->ai_family == AF_INET6)
setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, &one, sizeof(one));
#endif
if (bind(s, res->ai_addr, res->ai_addrlen) == -1) {
close(s);
return 0;
}
if (listen(s, 5) == -1) {
close(s);
return 0;
}
ls = xmalloc(sizeof(*ls));
ls->ls_fd = s;
LIST_INSERT_HEAD(&all_listener, ls, ls_link);
event_add(&ls->ls_event, s, 0, 1, cb, NULL);
return 1;
}
int
listen_sockaddr(const char *netaddr, void (*cb)(int, void *))
{
struct addrinfo *result, *res;
int got_address;
if ((result = prepare_getaddrinfo(netaddr, 0)) == NULL)
return -1;
got_address = 0;
for (res = result; res != NULL; res = res->ai_next)
got_address |= bind_and_listen(res, cb);
freeaddrinfo(result);
return got_address ? 0 : -1;
}

7
pkgtools/pbulk/files/pbulk/lib/pbulk.h
View file

@ -1,4 +1,4 @@
/* $NetBSD: pbulk.h,v 1.5 2014/01/15 13:52:10 joerg Exp $ */
/* $NetBSD: pbulk.h,v 1.6 2015/12/07 16:52:39 joerg Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -75,8 +75,9 @@ void deferred_write(int fd, const void *, size_t,
ssize_t atomic_read(int, void *, size_t);
ssize_t atomic_write(int, const void *, size_t);
int parse_sockaddr_in(const char *, struct sockaddr_in *);
int connect_sockaddr(const char *);
int listen_sockaddr(const char *, void (*)(int, void *));
void shutdown_listeners(void);
pid_t fork_chdir_exec(const char *, const char *,
const char * const *, int *);
char *read_from_child(const char *, const char *,

13
pkgtools/pbulk/files/pbulk/pbuild/client.c
View file

@ -1,4 +1,4 @@
/* $NetBSD: client.c,v 1.5 2015/10/21 23:03:17 joerg Exp $ */
/* $NetBSD: client.c,v 1.6 2015/12/07 16:52:39 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -52,20 +52,13 @@
void
client_mode(const char *client_port)
{
struct sockaddr_in dst;
uint32_t build_info_len;
ssize_t recv_bytes, sent_bytes;
char *build_info;
int fd;
if (parse_sockaddr_in(client_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not connect socket");
if ((fd = connect_sockaddr(client_port)) == -1)
err(1, "Could not creation connection to %s", client_port);
loop:
sent_bytes = atomic_write(fd, "G", 1);

37
pkgtools/pbulk/files/pbulk/pbuild/master.c
View file

@ -1,4 +1,4 @@
/* $NetBSD: master.c,v 1.10 2015/10/21 23:03:17 joerg Exp $ */
/* $NetBSD: master.c,v 1.11 2015/12/07 16:52:39 joerg Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -53,8 +53,6 @@
static int clients_started;
static LIST_HEAD(, build_peer) active_peers, inactive_peers, unassigned_peers;
static struct event listen_event;
static int listen_event_socket;
static struct signal_event child_event;
static pid_t child_pid;
@ -182,14 +180,16 @@ shutdown_master(void)
struct timeval tv;
struct build_peer *peer;
event_del(&listen_event);
(void)close(listen_event_socket);
shutdown_listeners();
LIST_FOREACH(peer, &inactive_peers, peer_link) {
uint32_t net_build_info_len = htonl(0);
(void)memcpy(peer->tmp_buf, &net_build_info_len, 4);
deferred_write(peer->fd, peer->tmp_buf, 4, peer, do_nothing,
kill_peer);
}
/* Give clients a second to close connections to prevent TIME_WAIT. */
tv.tv_sec = 1;
tv.tv_usec = 0;
event_loopexit(&tv);
@ -252,12 +252,9 @@ static void
listen_handler(int sock, void *arg)
{
struct build_peer *peer;
struct sockaddr_in src;
socklen_t src_len;
int fd;
src_len = sizeof(src);
if ((fd = accept(sock, (struct sockaddr *)&src, &src_len)) == -1) {
if ((fd = accept(sock, NULL, 0)) == -1) {
warn("Could not accept connection");
return;
}
@ -300,30 +297,14 @@ child_handler(struct signal_event *ev)
void
master_mode(const char *master_port, const char *start_script)
{
struct sockaddr_in dst;
int fd;
LIST_INIT(&active_peers);
LIST_INIT(&inactive_peers);
LIST_INIT(&unassigned_peers);
event_init();
if (parse_sockaddr_in(master_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
err(1, "Could not set close-on-exec flag");
if (bind(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not bind socket");
if (listen(fd, 5) == -1)
err(1, "Could not listen on socket");
event_add(&listen_event, fd, 0, 1, listen_handler, NULL);
listen_event_socket = fd;
if (listen_sockaddr(master_port, listen_handler))
errx(1, "Could not create listen socket for %s", master_port);
if (start_script) {
signal_add(&child_event, SIGCHLD, child_handler);
@ -339,6 +320,4 @@ master_mode(const char *master_port, const char *start_script)
}
event_dispatch();
(void)close(fd);
}

20
pkgtools/pbulk/files/pbulk/pbuild/pbulk-build.1
View file

@ -1,4 +1,4 @@
.\" $NetBSD: pbulk-build.1,v 1.1.1.1 2007/06/19 19:49:55 joerg Exp $
.\" $NetBSD: pbulk-build.1,v 1.2 2015/12/07 16:52:39 joerg Exp $
.\"
.\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
.\" All rights reserved.
@ -32,16 +32,16 @@
.Nd build all packages specified in input file
.Sh SYNOPSIS
.Nm
.Fl s Oo Ar ip: Oc Ns Ar port
.Fl s Oo Ar host: Oc Ns Ar port
.Nm
.Op Fl v
.Fl c Oo Ar ip: Oc Ns Ar port
.Fl c Oo Ar host: Oc Ns Ar port
.Fl b Ar build_script
.Nm
.Op Fl I Ar start_script
.Op Fl r Ar report_file
.Op Fl v
.Fl m Oo Ar ip: Oc Ns Ar port
.Fl m Oo Ar host: Oc Ns Ar port
.Ar input success error
.Nm
.Op Fl r Ar report_file
@ -61,11 +61,11 @@ for building the packages.
See
.Sx BUILD SCRIPT FORMAT
for details.
.It Fl c Oo Ar ip: Oc Ns Ar port
.It Fl c Oo Ar host: Oc Ns Ar port
Obtain jobs from master running on the given
.Ar port
(or
.Ar ip:port ) .
.Ar host:port ) .
If used with
.Fl v ,
print the name of the package to build to stdout.
@ -73,18 +73,18 @@ print the name of the package to build to stdout.
Run
.Ar start_script
after opening the socket and wait for completion before entering build loop.
.It Fl m Oo Ar ip: Oc Ns Ar port
.It Fl m Oo Ar host : Oc Ns Ar port
Enter master mode.
The master binds to
.Ar port
(or
.Ar ip:port )
.Ar host:port )
and waits for clients to connect and build individual packages.
.It Fl s Oo Ar ip: Oc Ns Ar port
.It Fl s Oo Ar host: Oc Ns Ar port
Query the master running on the given
.Ar port
(or
.Ar ip:port )
.Ar host:port )
for the current number of successful, open, and failed builds.
.It Fl r Ar report_file
Write name of each package,

13
pkgtools/pbulk/files/pbulk/pbuild/stat.c
View file

@ -1,4 +1,4 @@
/* $NetBSD: stat.c,v 1.3 2008/01/15 22:14:30 joerg Exp $ */
/* $NetBSD: stat.c,v 1.4 2015/12/07 16:52:39 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -52,21 +52,14 @@
void
stat_mode(const char *client_port)
{
struct sockaddr_in dst;
ssize_t recv_bytes, sent_bytes;
char buf[7 * 4];
struct build_stat st;
uint32_t tmp;
int fd;
if (parse_sockaddr_in(client_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not connect socket");
if ((fd = connect_sockaddr(client_port)) == -1)
err(1, "Could not creation connection to %s", client_port);
sent_bytes = write(fd, "S", 1);
if (sent_bytes == -1)

13
pkgtools/pbulk/files/pbulk/pscan/client.c
View file

@ -1,4 +1,4 @@
/* $NetBSD: client.c,v 1.4 2015/10/21 23:03:17 joerg Exp $ */
/* $NetBSD: client.c,v 1.5 2015/12/07 16:52:40 joerg Exp $ */
/*-
* Copyright (c) 2007 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -51,7 +51,6 @@
void
client_mode(const char *client_port)
{
struct sockaddr_in dst;
uint16_t path_len;
uint32_t net_output_len;
ssize_t recv_bytes, sent_bytes;
@ -59,14 +58,8 @@ client_mode(const char *client_port)
char *path, *output;
int fd;
if (parse_sockaddr_in(client_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (connect(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not connect socket");
if ((fd = connect_sockaddr(client_port)) == -1)
err(1, "Could not creation connection to %s", client_port);
loop:
recv_bytes = atomic_read(fd, &path_len, 2);

37
pkgtools/pbulk/files/pbulk/pscan/master.c
View file

@ -1,4 +1,4 @@
/* $NetBSD: master.c,v 1.10 2015/10/22 19:57:47 joerg Exp $ */
/* $NetBSD: master.c,v 1.11 2015/12/07 16:52:40 joerg Exp $ */
/*-
* Copyright (c) 2007, 2009 Joerg Sonnenberger <joerg@NetBSD.org>.
@ -53,8 +53,6 @@
static int clients_started;
static LIST_HEAD(, scan_peer) active_peers, inactive_peers;
static struct event listen_event;
static int listen_event_socket;
static struct signal_event child_event;
static pid_t child_pid;
@ -159,8 +157,8 @@ shutdown_master(void)
struct timeval tv;
struct scan_peer *peer;
event_del(&listen_event);
(void)close(listen_event_socket);
shutdown_listeners();
LIST_FOREACH(peer, &inactive_peers, peer_link) {
uint16_t net_job_len = htons(0);
(void)memcpy(peer->tmp_buf, &net_job_len, 2);
@ -168,6 +166,8 @@ shutdown_master(void)
deferred_write(peer->fd, peer->tmp_buf, 2, peer, do_nothing,
kill_peer);
}
/* Give clients a second to close connections to prevent TIME_WAIT. */
tv.tv_sec = 1;
tv.tv_usec = 0;
event_loopexit(&tv);
@ -205,12 +205,9 @@ static void
listen_handler(int sock, void *arg)
{
struct scan_peer *peer;
struct sockaddr_in src;
socklen_t src_len;
int fd;
src_len = sizeof(src);
if ((fd = accept(sock, (struct sockaddr *)&src, &src_len)) == -1) {
if ((fd = accept(sock, NULL, 0)) == -1) {
warn("Could not accept connection");
return;
}
@ -253,29 +250,13 @@ child_handler(struct signal_event *ev)
void
master_mode(const char *master_port, const char *start_script)
{
struct sockaddr_in dst;
int fd;
LIST_INIT(&active_peers);
LIST_INIT(&inactive_peers);
event_init();
if (parse_sockaddr_in(master_port, &dst))
errx(1, "Could not parse addr/port");
fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (fd == -1)
err(1, "Could not create socket");
if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1)
err(1, "Could not set close-on-exec flag");
if (bind(fd, (struct sockaddr *)&dst, sizeof(dst)) == -1)
err(1, "Could not bind socket");
if (listen(fd, 5) == -1)
err(1, "Could not listen on socket");
event_add(&listen_event, fd, 0, 1, listen_handler, NULL);
listen_event_socket = fd;
if (listen_sockaddr(master_port, listen_handler))
errx(1, "Could not create listen socket for %s", master_port);
if (start_script) {
signal_add(&child_event, SIGCHLD, child_handler);
@ -291,6 +272,4 @@ master_mode(const char *master_port, const char *start_script)
}
event_dispatch();
(void)close(fd);
}

14
pkgtools/pbulk/files/pbulk/pscan/pbulk-scan.1
View file

@ -1,4 +1,4 @@
.\" $NetBSD: pbulk-scan.1,v 1.1.1.1 2007/06/19 19:49:57 joerg Exp $
.\" $NetBSD: pbulk-scan.1,v 1.2 2015/12/07 16:52:40 joerg Exp $
.\"
.\" Copyright (c) 2007 Thomas Klausner and Joerg Sonnenberger.
.\" All rights reserved.
@ -33,13 +33,13 @@
.Sh SYNOPSIS
.Nm
.Op Fl v
.Fl c Oo Ar ip: Oc Ns Ar port
.Fl c Oo Ar host: Oc Ns Ar port
.Fl M Ar make
.Ar pkgsrc
.Nm
.Op Fl lv
.Op Fl I Ar start_script
.Op Fl m Oo Ar ip: Oc Ns Ar port
.Op Fl m Oo Ar host: Oc Ns Ar port
.Fl M Ar make
.Ar pkgsrc output
.Sh DESCRIPTION
@ -48,11 +48,11 @@ extracts information for a pbulk bulk build from a pkgsrc tree.
.Pp
Supported options are:
.Bl -tag -width 15n -offset indent
.It Fl c Oo Ar ip: Oc Ns Ar port
.It Fl c Oo Ar host: Oc Ns Ar port
Connect to pbulk bulk build master process on
.Ar port
(or
.Ar ip:port ) .
.Ar host:port ) .
.It Fl I Ar start_script
Run
.Ar start_script
@ -72,14 +72,14 @@ Usually
.Dq make
or
.Dq bmake .
.It Fl m Oo Ar ip: Oc Ns Ar port
.It Fl m Oo Ar host: Oc Ns Ar port
Enter master mode.
In this mode,
.Nm
waits for connections on
.Ar port
(or
.Ar ip:port ) .
.Ar host:port ) .
.It Fl v
Log each location to be scanned or other progress to stdout.
.El