On Thu, Feb 25, 2021 at 05:03:19PM +0100, Claudio Jeker wrote:
> On Fri, Feb 19, 2021 at 07:10:02PM +0100, Claudio Jeker wrote:
> > Some TAL files now include an https URI where the TA can be fetched from.
> > With this diff rpki-client will download the TA from https unless that
> > fails and then fall back to rsync.
> >
> > This is not yet perfect but the diff is already large enough (adding a
> > full event based https client based on ftp codebase). For RRDP more a lot
> > more is required and I probably will refactor the main.c code then.
> >
> > At the moment this adds a local mkostempat() function to implement
> > mkostemp() but with openat() instead of open(). I hope in the future libc
> > will provide something.
> >
> > Thanks in advance for all the feedback
>
> Updated diff. I found some bugs in the http.c code base regarding
> conncetion failures (because of unreachable IPv6 on the server) and fixed
> redirects. Those should now be fixed.
A couple of comments inline. Some cosmetic, some minor bugs.
Apart from a return value glitch for tls_close() the libtls handling
looks correct to me and while I'm not a http expert, the state machine
makes sense. The string handling (chopping up and re-encoding) looks
correct with appropriate bounds checking. I spent quite a bit of time
going through it.
Take from my comments whatever you like/makes sense, but I'm ok with
landing this and improving in tree.
>
> --
> :wq Claudio
>
> Index: Makefile
> ===================================================================
> RCS file: /cvs/src/usr.sbin/rpki-client/Makefile,v
> retrieving revision 1.18
> diff -u -p -r1.18 Makefile
> --- Makefile 4 Feb 2021 08:10:24 -0000 1.18
> +++ Makefile 19 Feb 2021 14:21:08 -0000
> @@ -1,13 +1,14 @@
> # $OpenBSD: Makefile,v 1.18 2021/02/04 08:10:24 claudio Exp $
>
> PROG= rpki-client
> -SRCS= as.c cert.c cms.c crl.c gbr.c io.c ip.c log.c main.c mft.c
> mkdir.c \
> - output.c output-bgpd.c output-bird.c output-csv.c output-json.c \
> - parser.c roa.c rsync.c tal.c validate.c x509.c
> +SRCS= as.c cert.c cms.c crl.c gbr.c http.c io.c ip.c log.c main.c
> mft.c \
> + mkdir.c mkostempat.c output.c output-bgpd.c output-bird.c \
> + output-csv.c output-json.c parser.c roa.c rsync.c tal.c validate.c \
> + x509.c
> MAN= rpki-client.8
>
> -LDADD+= -lcrypto -lutil
> -DPADD+= ${LIBCRYPTO} ${LIBUTIL}
> +LDADD+= -ltls -lssl -lcrypto -lutil
> +DPADD+= ${LIBTLS} ${LIBSSL} ${LIBCRYPTO} ${LIBUTIL}
>
> CFLAGS+= -Wall -I${.CURDIR}
> CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes
> Index: extern.h
> ===================================================================
> RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v
> retrieving revision 1.47
> diff -u -p -r1.47 extern.h
> --- extern.h 22 Feb 2021 09:46:05 -0000 1.47
> +++ extern.h 23 Feb 2021 10:36:38 -0000
> @@ -399,6 +399,10 @@ void proc_parser(int) __attribute__((n
> char *rsync_base_uri(const char *);
> void proc_rsync(char *, char *, int) __attribute__((noreturn));
>
> +/* Http-specific. */
> +
> +void proc_http(char *, int);
> +
> /* Logging (though really used for OpenSSL errors). */
>
> void cryptowarnx(const char *, ...)
> @@ -417,6 +421,7 @@ void io_str_buffer(struct ibuf *, cons
> void io_simple_read(int, void *, size_t);
> void io_buf_read_alloc(int, void **, size_t *);
> void io_str_read(int, char **);
> +int io_recvfd(int, void *, size_t);
>
> /* X509 helpers. */
>
> @@ -450,6 +455,7 @@ void logx(const char *fmt, ...)
> __attribute__((format(printf, 1, 2)));
>
> int mkpathat(int, const char *);
> +int mkostempat(int, char *, int);
>
> #define RPKI_PATH_OUT_DIR "/var/db/rpki-client"
> #define RPKI_PATH_BASE_DIR "/var/cache/rpki-client"
> Index: http.c
> ===================================================================
> RCS file: http.c
> diff -N http.c
> --- /dev/null 1 Jan 1970 00:00:00 -0000
> +++ http.c 24 Feb 2021 20:03:25 -0000
> @@ -0,0 +1,1226 @@
> +/* $OpenBSD$ */
> +/*
> + * Copyright (c) 2020 Nils Fisher <[email protected]>
> + * Copyright (c) 2020 Claudio Jeker <[email protected]>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +/*-
> + * Copyright (c) 1997 The NetBSD Foundation, Inc.
> + * All rights reserved.
> + *
> + * This code is derived from software contributed to The NetBSD Foundation
> + * by Jason Thorpe and Luke Mewburn.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions
> + * are met:
> + * 1. Redistributions of source code must retain the above copyright
> + * notice, this list of conditions and the following disclaimer.
> + * 2. Redistributions in binary form must reproduce the above copyright
> + * notice, this list of conditions and the following disclaimer in the
> + * documentation and/or other materials provided with the distribution.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
> + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> LIMITED
> + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
> + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
> + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
> + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
> + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
> + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
> + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
> + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
> + * POSSIBILITY OF SUCH DAMAGE.
> + */
> +#include <sys/types.h>
> +#include <sys/queue.h>
> +#include <sys/socket.h>
> +
> +#include <ctype.h>
> +#include <err.h>
> +#include <errno.h>
> +#include <limits.h>
> +#include <netdb.h>
> +#include <poll.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <unistd.h>
> +#include <vis.h>
> +#include <imsg.h>
> +
> +#include <tls.h>
> +
> +#include "extern.h"
> +
> +#define HTTP_USER_AGENT "OpenBSD rpki-client"
> +#define HTTP_BUF_SIZE (32 * 1024)
> +#define MAX_CONNECTIONS 12
> +
> +#define WANT_POLLIN 1
> +#define WANT_POLLOUT 2
> +
> +enum http_state {
> + STATE_FREE,
> + STATE_INIT,
> + STATE_CONNECT,
> + STATE_TLSCONNECT,
> + STATE_REQUEST,
> + STATE_RESPONSE_STATUS,
> + STATE_RESPONSE_HEADER,
> + STATE_RESPONSE_DATA,
> + STATE_RESPONSE_CHUNKED,
> + STATE_WRITE_DATA,
> + STATE_DONE,
> +};
> +
> +struct http_proxy {
> + char *proxyhost;
> + char *proxyuser;
> + char *proxypw;
> +};
> +
> +struct http_connection {
> + char *url;
> + char *host;
> + char *port;
> + char *path; /* points into url */
maybe make path const so it's clear that it's not meant for freeing?
> + char *modified_since;
> + char *last_modified;
> + struct addrinfo *res0;
> + struct addrinfo *res;
> + struct tls *tls;
> + char *buf;
> + size_t bufsz;
> + size_t bufpos;
> + size_t id;
> + size_t chunksz;
> + off_t filesize;
> + off_t bytes;
> + int status;
> + int redirect_loop;
> + int fd;
> + int outfd;
> + short events;
> + short chunked;
> + enum http_state state;
> +};
> +
> +struct msgbuf msgq;
> +struct sockaddr_storage http_bindaddr;
> +struct tls_config *tls_config;
> +uint8_t *tls_ca_mem;
> +size_t tls_ca_size;
> +
> +char *resp_buf[MAX_CONNECTIONS];
> +size_t resp_bsz[MAX_CONNECTIONS];
> +size_t resp_idx;
> +
> +/*
> + * Return a string that can be used in error message to identify the
> + * connection.
> + */
> +static const char *
> +http_info(const char *url)
> +{
> + static char buf[64];
> +
> + if (strnvis(buf, url, sizeof buf, VIS_SAFE) >= (int)sizeof buf) {
> + /* overflow, add indicator */
> + memcpy(buf + sizeof buf - 4, "...", 4);
> + }
> +
> + return buf;
> +}
> +
> +/*
> + * Determine whether the character needs encoding, per RFC1738:
> + * - No corresponding graphic US-ASCII.
> + * - Unsafe characters.
> + */
> +static int
> +unsafe_char(const char *c0)
> +{
> + const char *unsafe_chars = " <>\"#{}|\\^~[]`";
> + const unsigned char *c = (const unsigned char *)c0;
> +
> + /*
> + * No corresponding graphic US-ASCII.
> + * Control characters and octets not used in US-ASCII.
> + */
> + return (iscntrl(*c) || !isascii(*c) ||
> +
> + /*
> + * Unsafe characters.
> + * '%' is also unsafe, if is not followed by two
> + * hexadecimal digits.
> + */
> + strchr(unsafe_chars, *c) != NULL ||
> + (*c == '%' && (!isxdigit(*++c) || !isxdigit(*++c))));
> +}
> +
> +/*
> + * Encode given URL, per RFC1738.
> + * Allocate and return string to the caller.
> + */
> +static char *
> +url_encode(const char *path)
> +{
> + size_t i, length, new_length;
> + char *epath, *epathp;
> +
> + length = new_length = strlen(path);
> +
> + /*
> + * First pass:
> + * Count unsafe characters, and determine length of the
> + * final URL.
> + */
> + for (i = 0; i < length; i++)
> + if (unsafe_char(path + i))
> + new_length += 2;
> +
> + epath = epathp = malloc(new_length + 1); /* One more for '\0'. */
> + if (epath == NULL)
> + err(1, NULL);
> +
> + /*
> + * Second pass:
> + * Encode, and copy final URL.
> + */
> + for (i = 0; i < length; i++)
> + if (unsafe_char(path + i)) {
> + snprintf(epathp, 4, "%%" "%02x",
> + (unsigned char)path[i]);
> + epathp += 3;
> + } else
> + *(epathp++) = path[i];
> +
> + *epathp = '\0';
> + return (epath);
> +}
> +
> +static void
> +http_setup(void)
> +{
> + tls_config = tls_config_new();
> + if (tls_config == NULL)
> + errx(1, "tls config failed");
> +#if 0
> + /* TODO Should we allow extra protos and ciphers? */
> + if (tls_config_set_protocols(tls_config,
> + TLS_PROTOCOLS_ALL) != 0)
libtls returns -1 or NULL on error, so I would check for -1.
I won't annotate this further.
> + errx(1, "tls set protocols failed: %s",
> + tls_config_error(tls_config));
> + if (tls_config_set_ciphers(tls_config, "legacy") != 0)
> + errx(1, "tls set ciphers failed: %s",
> + tls_config_error(tls_config));
> +#endif
> +
> + /* load cert file from disk now */
> + tls_ca_mem = tls_load_file(tls_default_ca_cert_file(),
> + &tls_ca_size, NULL);
> + if (tls_ca_mem == NULL)
> + err(1, "tls_load_file");
> + tls_config_set_ca_mem(tls_config, tls_ca_mem, tls_ca_size);
> +
> + /* TODO initalize proxy settings */
> +
> +}
> +
> +static int
> +http_resolv(struct http_connection *conn, const char *host, const char *port)
> +{
> + struct addrinfo hints;
> + int error;
> +
> + memset(&hints, 0, sizeof(hints));
> + hints.ai_family = PF_UNSPEC;
> + hints.ai_socktype = SOCK_STREAM;
> + error = getaddrinfo(host, port, &hints, &conn->res0);
> + /*
> + * If the services file is corrupt/missing, fall back
> + * on our hard-coded defines.
> + */
> + if (error == EAI_SERVICE)
> + error = getaddrinfo(host, "443", &hints, &conn->res0);
> + if (error) {
error != NULL
> + warnx("%s: %s", host, gai_strerror(error));
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> +static void
> +http_done(struct http_connection *conn, int ok)
> +{
> + struct ibuf *b;
> +
> + conn->state = STATE_DONE;
> +
> + if ((b = ibuf_dynamic(64, UINT_MAX)) == NULL)
> + err(1, NULL);
> + io_simple_buffer(b, &conn->id, sizeof(conn->id));
> + io_simple_buffer(b, &ok, sizeof(ok));
> +#if 0 /* TODO: cache last_modified */
> + io_str_buffer(b, conn->last_modified);
> +#endif
> + ibuf_close(&msgq, b);
> +}
> +
> +static void
> +http_fail(size_t id)
> +{
> + struct ibuf *b;
> + int ok = 0;
> +
> + if ((b = ibuf_dynamic(8, UINT_MAX)) == NULL)
> + err(1, NULL);
> + io_simple_buffer(b, &id, sizeof(id));
> + io_simple_buffer(b, &ok, sizeof(ok));
> + ibuf_close(&msgq, b);
> +}
> +
> +static void
> +http_free(struct http_connection *conn)
> +{
> + if (conn->state != STATE_DONE)
> + http_fail(conn->id);
> +
> + free(conn->host);
> + free(conn->port);
> + free(conn->url);
I would move free(conn->url) to the top so you free stuff in the order of the
struct members.
> + /* no need to free conn->path it points into conn->uri */
typo: conn->url
> + free(conn->modified_since);
> + free(conn->last_modified);
> + free(conn->buf);
> +
> + if (conn->res0)
> + freeaddrinfo(conn->res0);
> +
> + tls_free(conn->tls);
> + conn->tls = NULL;
No need for setting this to NULL.
> +
> + if (conn->fd != -1)
> + close(conn->fd);
> + close(conn->outfd);
> + free(conn);
> +}
> +
> +static int
> +http_close(struct http_connection *conn)
> +{
> + if (conn->tls) {
The file is a bit inconsistent with comparisons to NULL.
I would prefer if they were done all explicitly but will stop annotating
this here.
> + switch (tls_close(conn->tls)) {
> + case TLS_WANT_POLLIN:
> + return WANT_POLLIN;
> + case TLS_WANT_POLLOUT:
> + return WANT_POLLOUT;
Looks like http_close() failure is currently silent, otherwise the missing
success case would have been noticed:
case 0:
return 0;
> + case -1:
> + warnx("%s: TLS close: %s", http_info(conn->url),
> + tls_error(conn->tls));
> + return -1;
> + }
> + }
> +
> + return -1;
> +}
> +
> +static int
> +http_parse_uri(char *uri, char **ohost, char **oport, char **opath)
> +{
> + char *host, *port = NULL, *path;
> + char *hosttail;
> +
> + if (strncasecmp(uri, "https://";, 8) != 0) {
> + warnx("%s: not using https schema", http_info(uri));
> + return -1;
> + }
> + host = uri + 8;
> + if ((path = strchr(host, '/')) == NULL) {
> + warnx("%s: missing https path", http_info(uri));
> + return -1;
> + }
> + if (path - uri > INT_MAX - 1) {
> + warnx("%s: preposterous host length", http_info(uri));
> + return -1;
> + }
> + if (*host == '[') {
> + char *scope;
> + if ((hosttail = memrchr(host, ']', path - host)) != NULL &&
> + (hosttail[1] == '/' || hosttail[1] == ':'))
> + host++;
> + if (hosttail[1] == ':')
> + port = hosttail + 1;
> + if ((scope = memchr(host, '%', hosttail - host)) != NULL)
> + hosttail = scope;
> + } else {
> + if ((hosttail = memrchr(host, ':', path - host)) != NULL)
> + port = hosttail + 1;
> + else
> + hosttail = path;
> + }
> +
> + if ((host = strndup(host, hosttail - host)) == NULL)
> + err(1, "strndup");
> + if (port) {
> + if ((port = strndup(port, path - port)) == NULL)
> + err(1, "strndup");
> + } else {
> + if ((port = strdup("443")) == NULL)
> + err(1, "strdup");
> + }
> + /* do not include the initial / in path */
> + path++;
> +
> + *ohost = host;
> + *oport = port;
> + *opath = path;
> +
> + return 0;
> +}
> +
> +
> +static struct http_connection *
> +http_new(size_t id, char *uri, char *modified_since, int outfd)
> +{
> + struct http_connection *conn;
> + char *host, *port, *path;
> +
> + if (http_parse_uri(uri, &host, &port, &path) == -1) {
> + free(uri);
> + free(modified_since);
> + return NULL;
> + }
> +
> + if ((conn = calloc(1, sizeof(*conn))) == NULL)
> + err(1, NULL);
> +
> + conn->id = id;
> + conn->fd = -1;
> + conn->outfd = outfd;
> + conn->host = host;
> + conn->port = port;
> + conn->path = path;
> + conn->url = uri;
> + conn->modified_since = modified_since;
> + conn->state = STATE_INIT;
> +
> + /* TODO proxy support (overload of host and port) */
> +
> + if (http_resolv(conn, host, port) == -1) {
> + http_free(conn);
> + return NULL;
> + }
> +
> + return conn;
> +}
> +
> +static int
> +http_redirect(struct http_connection *conn, char *uri)
> +{
> + char *host, *port, *path;
> +
> + warnx("redirect to %s", http_info(uri));
> +
> + if (http_parse_uri(uri, &host, &port, &path) == -1) {
> + free(uri);
> + return -1;
> + }
> +
> + free(conn->host);
> + conn->host = host;
> + free(conn->port);
> + conn->port = port;
> + free(conn->url);
> + conn->url = uri;
move handling of url to the beginning
> + conn->path = path;
conn->modified_since omitted?
> + free(conn->last_modified);
> + conn->last_modified = NULL;
> + free(conn->buf);
> + conn->buf = NULL;
> + conn->bufpos = 0;
> + conn->bufsz = 0;
> + tls_close(conn->tls);
> + tls_free(conn->tls);
> + conn->tls = NULL;
> + close(conn->fd);
> + conn->state = STATE_INIT;
> +
> + /* TODO proxy support (overload of host and port) */
> +
> + if (http_resolv(conn, host, port) == -1)
> + return -1;
> +
> + return 0;
> +}
> +
> +static int
> +http_connect(struct http_connection *conn)
> +{
> + char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST];
> + char *cause = "unknown";
> +
> + if (conn->fd != -1) {
> + close(conn->fd);
> + conn->fd = -1;
> + }
> +
> + /* start the loop below with first or next address */
> + if (conn->res == NULL)
> + conn->res = conn->res0;
> + else
> + conn->res = conn->res->ai_next;
> + for (; conn->res != NULL; conn->res = conn->res->ai_next) {
> + struct addrinfo *res = conn->res;
> + int fd, error, save_errno;
> +
> + if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf,
> + sizeof(hbuf), pbuf, sizeof(pbuf),
> + NI_NUMERICHOST | NI_NUMERICSERV) != 0)
> + strlcpy(hbuf, "(unknown)", sizeof(hbuf));
> +
> + fd = socket(res->ai_family,
> + res->ai_socktype | SOCK_NONBLOCK, res->ai_protocol);
> + if (fd == -1) {
> + cause = "socket";
> + continue;
> + }
> + conn->fd = fd;
> +
> + if (http_bindaddr.ss_family == res->ai_family) {
> + if (bind(conn->fd, (struct sockaddr *)&http_bindaddr,
> + res->ai_addrlen) == -1)
> + warn("%s: bind", http_info(conn->url));
> + }
> +
> + error = connect(conn->fd, res->ai_addr, res->ai_addrlen);
> + if (error == -1) {
> + if (errno == EINPROGRESS) {
> + /* waiting for connect to finish. */
> + return WANT_POLLOUT;
> + } else {
> + save_errno = errno;
> + close(conn->fd);
> + conn->fd = -1;
> + errno = save_errno;
> + cause = "connect";
> + continue;
> + }
> + }
> +
> + /* TODO proxy connect */
> +#if 0
> + if (proxyenv)
> + proxy_connect(conn->fd, sslhost, proxy_credentials); */
> +#endif
> + }
> + freeaddrinfo(conn->res0);
> + conn->res0 = NULL;
> + if (conn->fd == -1) {
> + warn("%s: %s", http_info(conn->url), cause);
> + return -1;
> + }
> + return 0;
> +}
> +
> +static int
> +http_finish_connect(struct http_connection *conn)
> +{
> + int error = 0;
> + socklen_t len;
> +
> + len = sizeof(error);
> + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &error, &len) == -1) {
> + warn("%s: getsockopt SO_ERROR", http_info(conn->url));
> + /* connection will be closed by http_connect() */
> + return -1;
> + }
> + if (error != 0) {
> + errno = error;
> + warn("%s: connect", http_info(conn->url));
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> +static int
> +http_tls_handshake(struct http_connection *conn)
> +{
> + switch (tls_handshake(conn->tls)) {
> + case 0:
> + return 0;
> + case TLS_WANT_POLLIN:
> + return WANT_POLLIN;
> + case TLS_WANT_POLLOUT:
> + return WANT_POLLOUT;
> + }
> + warnx("%s: TLS handshake: %s", http_info(conn->url),
> + tls_error(conn->tls));
> + return -1;
> +}
> +
> +static int
> +http_tls_connect(struct http_connection *conn)
> +{
> + if ((conn->tls = tls_client()) == NULL) {
> + warn("tls_client");
> + return -1;
> + }
> + if (tls_configure(conn->tls, tls_config) != 0) {
> + warnx("%s: TLS configuration: %s\n", http_info(conn->url),
> + tls_error(conn->tls));
> + return -1;
> + }
> + if (tls_connect_socket(conn->tls, conn->fd, conn->host) != 0) {
> + warnx("%s: TLS connect: %s\n", http_info(conn->url),
> + tls_error(conn->tls));
> + return -1;
> + }
> + return http_tls_handshake(conn);
> +}
> +
> +static int
> +http_request(struct http_connection *conn)
> +{
> + char *host, *epath, *modified_since;
> + int with_port = 0;
> +
> + /* TODO adjust request for HTTP proxy setups */
> +
> + /*
> + * Send port number only if it's specified and does not equal
> + * the default. Some broken HTTP servers get confused if you explicitly
> + * send them the port number.
> + */
> + if (conn->port && strcmp(conn->port, "443") != 0)
> + with_port = 1;
> +
> + /* Construct the Host header from host and port info */
> + if (strchr(conn->host, ':')) {
> + if (asprintf(&host, "[%s]%s%s", conn->host,
> + with_port ? ":" : "", with_port ? conn->port : "") == -1)
> + err(1, NULL);
> +
> + } else {
> + if (asprintf(&host, "%s%s%s", conn->host,
> + with_port ? ":" : "", with_port ? conn->port : "") == -1)
> + err(1, NULL);
> + }
> +
> + /*
> + * Construct and send the request. Proxy requests don't want leading /.
> + */
> + epath = url_encode(conn->path);
> +
> + modified_since = NULL;
> + if (conn->modified_since) {
> + if (asprintf(&modified_since, "If-Modified-Since: %s\r\n",
> + conn->modified_since) == -1)
> + err(1, NULL);
> + }
> +
> + free(conn->buf);
> + conn->bufpos = 0;
> + if ((conn->bufsz = asprintf(&conn->buf,
> + "GET /%s HTTP/1.1\r\n"
> + "Connection: close\r\n"
> + "User-Agent: " HTTP_USER_AGENT "\r\n"
> + "Host: %s\r\n%s\r\n",
> + epath, host,
> + modified_since ? modified_since : "")) == -1)
> + err(1, NULL);
> +
> + free(epath);
> + free(host);
> + free(modified_since);
> +
> + return WANT_POLLOUT;
> +}
> +
> +static int
> +http_write(struct http_connection *conn)
> +{
> + ssize_t s;
> +
> + s = tls_write(conn->tls, conn->buf + conn->bufpos,
> + conn->bufsz - conn->bufpos);
> + if (s == -1) {
> + warnx("%s: TLS write: %s", http_info(conn->url),
> + tls_error(conn->tls));
> + return -1;
> + } else if (s == TLS_WANT_POLLIN) {
> + return WANT_POLLIN;
> + } else if (s == TLS_WANT_POLLOUT) {
> + return WANT_POLLOUT;
> + }
> +
> + conn->bufpos += s;
> + if (conn->bufpos == conn->bufsz)
> + return 0;
> + return WANT_POLLOUT;
> +}
> +
> +static int
> +http_parse_status(struct http_connection *conn, char *buf)
> +{
> + const char *errstr;
> + char *cp, ststr[4];
> + char gerror[200];
> + int status;
> +
> + cp = strchr(buf, ' ');
> + if (cp == NULL) {
> + warnx("Improper response from %s", http_info(conn->url));
> + return -1;
> + } else
> + cp++;
> +
> + strlcpy(ststr, cp, sizeof(ststr));
> + status = strtonum(ststr, 200, 599, &errstr);
> + if (errstr) {
> + strnvis(gerror, cp, sizeof gerror, VIS_SAFE);
> + warnx("Error retrieving %s: %s", http_info(conn->url), gerror);
> + return -1;
> + }
> +
> + switch (status) {
> + case 301:
> + case 302:
> + case 303:
> + case 307:
> + if (conn->redirect_loop++ > 10) {
> + warnx("%s: Too many redirections requested",
> + http_info(conn->url));
> + return -1;
> + }
> + /* FALLTHROUGH */
> + case 200:
> + case 206:
> + case 304:
> + conn->status = status;
> + break;
> + default:
> + strnvis(gerror, cp, sizeof gerror, VIS_SAFE);
> + warnx("Error retrieving %s: %s", http_info(conn->url), gerror);
> + break;
> + }
> +
> + return 0;
> +}
> +
> +static inline int
> +http_isredirect(struct http_connection *conn)
> +{
> + if ((conn->status >= 301 && conn->status <= 303) ||
> + conn->status == 307)
> + return 1;
> + return 0;
> +}
> +
> +static int
> +http_parse_header(struct http_connection *conn, char *buf)
> +{
> +#define CONTENTLEN "Content-Length: "
> +#define LOCATION "Location: "
> +#define TRANSFER_ENCODING "Transfer-Encoding: "
> +#define LAST_MODIFIED "Last-Modified: "
> + const char *errstr;
> + char *cp, *redirurl;
> + char *locbase, *loctail;
> +
> + cp = buf;
> + /* empty line, end of header */
> + if (*cp == '\0')
> + return 0;
> + else if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) {
> + size_t s;
> + cp += sizeof(CONTENTLEN) - 1;
> + if ((s = strcspn(cp, " \t")))
> + *(cp+s) = 0;
> + conn->filesize = strtonum(cp, 0, LLONG_MAX, &errstr);
> + if (errstr != NULL) {
> + warnx("Improper response from %s",
> + http_info(conn->url));
> + return -1;
> + }
> + } else if (http_isredirect(conn) &&
> + strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) {
> + cp += sizeof(LOCATION) - 1;
> + /*
> + * If there is a colon before the first slash, this URI
> + * is not relative. RFC 3986 4.2
> + */
> + if (cp[strcspn(cp, ":/")] != ':') {
> + /* XXX doesn't handle protocol-relative URIs */
> + if (*cp == '/') {
> + locbase = NULL;
> + cp++;
> + } else {
> + locbase = strdup(conn->path);
> + if (locbase == NULL)
> + err(1, "");
> + loctail = strchr(locbase, '#');
> + if (loctail != NULL)
> + *loctail = '\0';
> + loctail = strchr(locbase, '?');
> + if (loctail != NULL)
> + *loctail = '\0';
> + loctail = strrchr(locbase, '/');
> + if (loctail == NULL) {
> + free(locbase);
> + locbase = NULL;
> + } else
> + loctail[1] = '\0';
> + }
> + /* Contruct URL from relative redirect */
Typo: Construct
> + if (asprintf(&redirurl, "%.*s/%s%s",
> + (int)(conn->path - conn->url), conn->url,
> + locbase ? locbase : "",
> + cp) == -1)
> + err(1, "Cannot build redirect URL");
> + free(locbase);
> + } else if ((redirurl = strdup(cp)) == NULL)
> + err(1, "Cannot build redirect URL");
> + loctail = strchr(redirurl, '#');
> + if (loctail != NULL)
> + *loctail = '\0';
> + return http_redirect(conn, redirurl);
> + } else if (strncasecmp(cp, TRANSFER_ENCODING,
> + sizeof(TRANSFER_ENCODING) - 1) == 0) {
> + cp += sizeof(TRANSFER_ENCODING) - 1;
> + cp[strcspn(cp, " \t")] = '\0';
> + if (strcasecmp(cp, "chunked") == 0)
> + conn->chunked = 1;
> + } else if (strncasecmp(cp, LAST_MODIFIED,
> + sizeof(LAST_MODIFIED) - 1) == 0) {
> + cp += sizeof(LAST_MODIFIED) - 1;
> + conn->last_modified = strdup(cp);
> + }
> +
> + return 1;
> +}
> +
> +static char *
> +http_get_line(struct http_connection *conn)
> +{
> + char *end, *line;
> + size_t len;
> +
> + end = memchr(conn->buf, '\n', conn->bufpos);
> + if (end == NULL)
> + return NULL;
> +
> + len = end - conn->buf;
> + while (len > 0 && conn->buf[len - 1] == '\r')
> + --len;
> + line = strndup(conn->buf, len);
Should strndup failure not be a fatal error?
> +
> + /* consume line including \n */
> + end++;
> + conn->bufpos -= end - conn->buf;
> + memmove(conn->buf, end, conn->bufpos);
> +
> + return line;
> +}
> +
> +static int
> +http_parse_chunked(struct http_connection *conn, char *buf)
> +{
> + char *header = buf;
> + char *end;
> + int chunksize;
> +
> + /* ignore empty lines, used between chunk and next header */
> + if (*header == '\0')
> + return 1;
> +
> + /* strip CRLF and any optional chunk extension */
> + header[strcspn(header, ";\r\n")] = '\0';
> + errno = 0;
> + chunksize = strtoul(header, &end, 16);
> + if (errno || header[0] == '\0' || *end != '\0' ||
> + chunksize > INT_MAX) {
> + warnx("%s: Invalid chunk size", http_info(conn->url));
> + return -1;
> + }
> + conn->chunksz = chunksize;
> +
> + if (conn->chunksz == 0) {
> + http_done(conn, 1);
> + return 0;
> + }
> +
> + return 1;
> +}
> +
> +static int
> +http_read(struct http_connection *conn)
> +{
> + ssize_t s;
> + char *buf;
> +
> + s = tls_read(conn->tls, conn->buf + conn->bufpos,
> + conn->bufsz - conn->bufpos);
> + if (s == -1) {
> + warn("%s: TLS read: %s", http_info(conn->url),
> + tls_error(conn->tls));
> + return -1;
> + } else if (s == TLS_WANT_POLLIN) {
> + return WANT_POLLIN;
> + } else if (s == TLS_WANT_POLLOUT) {
> + return WANT_POLLOUT;
> + }
> +
> + if (s == 0 && conn->bufpos == 0) {
> + warnx("%s: short read, connection closed",
> + http_info(conn->url));
> + return -1;
> + }
> +
> + conn->bufpos += s;
> +
> + switch (conn->state) {
> + case STATE_RESPONSE_STATUS:
> + buf = http_get_line(conn);
> + if (buf == NULL)
> + return WANT_POLLIN;
> + if (http_parse_status(conn, buf) == -1) {
> + free(buf);
> + return -1;
> + }
> + free(buf);
> + conn->state = STATE_RESPONSE_HEADER;
> + /* FALLTHROUGH */
> + case STATE_RESPONSE_HEADER:
> + while ((buf = http_get_line(conn)) != NULL) {
> + int rv;
> +
> + rv = http_parse_header(conn, buf);
> + free(buf);
> + if (rv != 1)
> + return rv;
> + }
> + return WANT_POLLIN;
> + case STATE_RESPONSE_DATA:
> + if (conn->bufpos == conn->bufsz ||
> + conn->filesize - conn->bytes <= (off_t)conn->bufpos)
> + return 0;
> + return WANT_POLLIN;
> + case STATE_RESPONSE_CHUNKED:
> + while (conn->chunksz == 0) {
> + buf = http_get_line(conn);
> + if (buf == NULL)
> + return WANT_POLLIN;
> + switch (http_parse_chunked(conn, buf)) {
> + case -1:
> + free(buf);
> + return -1;
> + case 0:
> + free(buf);
> + return 0;
> + }
> + free(buf);
> + }
> +
> + if (conn->bufpos == conn->bufsz ||
> + conn->chunksz <= conn->bufpos)
> + return 0;
> + return WANT_POLLIN;
> + default:
> + errx(1, "unexpected http state");
> + }
> +}
> +
> +static int
> +data_write(struct http_connection *conn)
> +{
> + ssize_t s;
> + size_t bsz = conn->bufpos;
> +
> + if (conn->filesize - conn->bytes < (off_t)bsz)
> + bsz = conn->filesize - conn->bytes;
> +
> + s = write(conn->outfd, conn->buf, bsz);
> + if (s == -1) {
> + warn("%s: Data write", http_info(conn->url));
> + return -1;
> + }
> +
> + conn->bufpos -= s;
> + conn->bytes += s;
> + memmove(conn->buf, conn->buf + s, conn->bufpos);
> +
> + if (conn->bytes == conn->filesize) {
> + http_done(conn, 1);
> + return 0;
> + }
> +
> + if (conn->bufpos == 0)
> + return 0;
> +
> + return WANT_POLLOUT;
> +}
> +
> +static int
> +chunk_write(struct http_connection *conn)
> +{
> + ssize_t s;
> + size_t bsz = conn->bufpos;
> +
> + if (bsz > conn->chunksz)
> + bsz = conn->chunksz;
> +
> + s = write(conn->outfd, conn->buf, bsz);
> + if (s == -1) {
> + warn("%s: Chunk write", http_info(conn->url));
> + return -1;
> + }
> +
> + conn->bufpos -= s;
> + conn->chunksz -= s;
> + conn->bytes += s;
> + memmove(conn->buf, conn->buf + s, conn->bufpos);
> +
> + if (conn->bufpos == 0 || conn->chunksz == 0)
> + return 0;
> +
> + return WANT_POLLOUT;
> +}
> +
> +static int
> +http_handle(struct http_connection *conn)
> +{
> + switch (conn->state) {
> + case STATE_INIT:
> + return 0;
> + case STATE_CONNECT:
> + if (http_finish_connect(conn) == -1)
> + /* something went wrong, try other host */
> + return http_connect(conn);
> + return 0;
> + case STATE_TLSCONNECT:
> + return http_tls_handshake(conn);
> + case STATE_REQUEST:
> + return http_write(conn);
> + case STATE_RESPONSE_STATUS:
> + case STATE_RESPONSE_HEADER:
> + case STATE_RESPONSE_DATA:
> + case STATE_RESPONSE_CHUNKED:
> + return http_read(conn);
> + case STATE_WRITE_DATA:
> + if (conn->chunked)
> + return chunk_write(conn);
> + else
> + return data_write(conn);
> + case STATE_DONE:
> + return http_close(conn);
> + case STATE_FREE:
> + errx(1, "bad http state");
> + }
> +}
> +
> +static int
> +http_nextstep(struct http_connection *conn)
> +{
> + switch (conn->state) {
> + case STATE_INIT:
> + conn->state = STATE_CONNECT;
> + return http_connect(conn);
> + case STATE_CONNECT:
> + conn->state = STATE_TLSCONNECT;
> + return http_tls_connect(conn);
> + case STATE_TLSCONNECT:
> + conn->state = STATE_REQUEST;
> + return http_request(conn);
> + case STATE_REQUEST:
> + conn->state = STATE_RESPONSE_STATUS;
> + free(conn->buf);
> + /* allocate the read buffer */
> + if ((conn->buf = malloc(HTTP_BUF_SIZE)) == NULL)
> + err(1, NULL);
> + conn->bufpos = 0;
> + conn->bufsz = HTTP_BUF_SIZE;
> + return WANT_POLLIN;
> + case STATE_RESPONSE_STATUS:
> + conn->state = STATE_RESPONSE_HEADER;
> + return WANT_POLLIN;
> + case STATE_RESPONSE_HEADER:
> + if (conn->status == 200)
> + conn->state = STATE_RESPONSE_DATA;
> + else {
> + http_done(conn, 0);
> + return http_close(conn);
> + }
> + return WANT_POLLIN;
> + case STATE_RESPONSE_DATA:
> + conn->state = STATE_WRITE_DATA;
> + return WANT_POLLOUT;
> + case STATE_RESPONSE_CHUNKED:
> + conn->state = STATE_WRITE_DATA;
> + return WANT_POLLOUT;
> + case STATE_WRITE_DATA:
> + if (conn->chunked)
> + conn->state = STATE_RESPONSE_CHUNKED;
> + else
> + conn->state = STATE_RESPONSE_DATA;
> + return WANT_POLLIN;
> + case STATE_DONE:
> + return http_close(conn);
> + case STATE_FREE:
> + errx(1, "bad http state");
> + }
> +}
> +
> +static int
> +http_do(struct http_connection *conn)
> +{
> + switch (http_handle(conn)) {
> + case -1:
> + /* connection failure */
> + http_free(conn);
> + return -1;
> + case 0:
> + switch (http_nextstep(conn)) {
> + case WANT_POLLIN:
> + conn->events = POLLIN;
> + break;
> + case WANT_POLLOUT:
> + conn->events = POLLOUT;
> + break;
> + case -1:
> + http_free(conn);
> + return -1;
> + }
> + break;
> + case WANT_POLLIN:
> + conn->events = POLLIN;
> + break;
> + case WANT_POLLOUT:
> + conn->events = POLLOUT;
> + break;
> + }
> + return 0;
> +}
> +
> +void
> +proc_http(char *bind_addr, int fd)
> +{
> + struct http_connection *http_conns[MAX_CONNECTIONS];
> + struct pollfd pfds[MAX_CONNECTIONS + 1];
> + size_t i;
> + int active_connections;
> +
> + if (bind_addr) {
> + struct addrinfo hints, *res;
> +
> + bzero(&hints, sizeof(hints));
> + hints.ai_family = AF_UNSPEC;
> + hints.ai_socktype = SOCK_DGRAM; /*dummy*/
> + hints.ai_flags = AI_NUMERICHOST;
> + if (getaddrinfo(bind_addr, NULL, &hints, &res) == 0) {
> + memcpy(&http_bindaddr, res->ai_addr, res->ai_addrlen);
> + freeaddrinfo(res);
> + }
> + }
> + http_setup();
> +
> + if (pledge("stdio inet dns recvfd", NULL) == -1)
> + err(1, "pledge");
> +
> + memset(&http_conns, 0, sizeof(http_conns));
> + memset(&pfds, 0, sizeof(pfds));
> + pfds[MAX_CONNECTIONS].fd = fd;
> +
> + msgbuf_init(&msgq);
> + msgq.fd = fd;
> +
> + for (;;) {
> + active_connections = 0;
> + for (i = 0; i < MAX_CONNECTIONS; i++) {
> + struct http_connection *conn = http_conns[i];
> + if (conn == NULL) {
> + pfds[i].fd = -1;
> + continue;
> + }
> + if (conn->state == STATE_WRITE_DATA)
> + pfds[i].fd = conn->outfd;
> + else
> + pfds[i].fd = conn->fd;
> +
> + pfds[i].events = conn->events;
> + active_connections++;
> + }
> + pfds[MAX_CONNECTIONS].events = 0;
> + if (active_connections < MAX_CONNECTIONS)
> + pfds[MAX_CONNECTIONS].events |= POLLIN;
> + if (msgq.queued)
> + pfds[MAX_CONNECTIONS].events |= POLLOUT;
> +
> + if (poll(pfds, sizeof(pfds) / sizeof(pfds[0]), INFTIM) == -1)
> + err(1, "poll");
> +
> + if (pfds[MAX_CONNECTIONS].revents & POLLHUP)
> + break;
> +
> + if (pfds[MAX_CONNECTIONS].revents & POLLIN) {
> + struct http_connection *h;
> + size_t id;
> + int outfd;
> + char *uri;
> + char *mod;
> +
> + outfd = io_recvfd(fd, &id, sizeof(id));
> + io_str_read(fd, &uri);
> + io_str_read(fd, &mod);
> +
> + h = http_new(id, uri, mod, outfd);
> + if (h == NULL) {
> + close(outfd);
> + http_fail(id);
> + } else
> + for (i = 0; i < MAX_CONNECTIONS; i++) {
> + if (http_conns[i] != NULL)
> + continue;
> + http_conns[i] = h;
> + if (http_do(h) == -1)
> + http_conns[i] = NULL;
> + break;
> + }
> + }
> + if (pfds[MAX_CONNECTIONS].revents & POLLOUT) {
> + switch (msgbuf_write(&msgq)) {
> + case 0:
> + errx(1, "write: connection closed");
> + case -1:
> + err(1, "write");
> + }
> + }
> + for (i = 0; i < MAX_CONNECTIONS; i++) {
> + struct http_connection *conn = http_conns[i];
> +
> + if (conn == NULL)
> + continue;
> + /* event not ready */
> + if (!(pfds[i].revents & (conn->events | POLLHUP)))
> + continue;
> +
> + if (http_do(conn) == -1)
> + http_conns[i] = NULL;
> + }
> + }
> +}
> Index: io.c
> ===================================================================
> RCS file: /cvs/src/usr.sbin/rpki-client/io.c,v
> retrieving revision 1.12
> diff -u -p -r1.12 io.c
> --- io.c 8 Jan 2021 08:09:07 -0000 1.12
> +++ io.c 18 Feb 2021 09:54:16 -0000
> @@ -16,9 +16,11 @@
> */
>
> #include <sys/queue.h>
> +#include <sys/socket.h>
>
> #include <assert.h>
> #include <err.h>
> +#include <errno.h>
> #include <fcntl.h>
> #include <stdint.h>
> #include <stdlib.h>
> @@ -145,4 +147,79 @@ io_str_read(int fd, char **res)
> if ((*res = calloc(sz + 1, 1)) == NULL)
> err(1, NULL);
> io_simple_read(fd, *res, sz);
> +}
> +
> +/*
> + * Read data from socket but receive a file descriptor at the same time.
> + */
> +int
> +io_recvfd(int fd, void *res, size_t sz)
> +{
> + struct iovec iov;
> + struct msghdr msg;
> + struct cmsghdr *cmsg;
> + union {
> + struct cmsghdr hdr;
> + char buf[CMSG_SPACE(sizeof(int))];
> + } cmsgbuf;
> + int outfd = -1;
> + char *b = res;
> + ssize_t n;
> +
> + memset(&msg, 0, sizeof(msg));
> + memset(&cmsgbuf, 0, sizeof(cmsgbuf));
> +
> + iov.iov_base = res;
> + iov.iov_len = sz;
> +
> + msg.msg_iov = &iov;
> + msg.msg_iovlen = 1;
> + msg.msg_control = &cmsgbuf.buf;
> + msg.msg_controllen = sizeof(cmsgbuf.buf);
> +
> + while ((n = recvmsg(fd, &msg, 0)) == -1) {
> + if (errno == EINTR)
> + continue;
> + err(1, "recvmsg");
> + }
> +
> + if (n == 0)
> + errx(1, "recvmsg: unexpected end of file");
> +
> + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
> + cmsg = CMSG_NXTHDR(&msg, cmsg)) {
> + if (cmsg->cmsg_level == SOL_SOCKET &&
> + cmsg->cmsg_type == SCM_RIGHTS) {
> + int i, j, f;
> +
> + j = ((char *)cmsg + cmsg->cmsg_len -
> + (char *)CMSG_DATA(cmsg)) / sizeof(int);
> + for (i = 0; i < j; i++) {
> + f = ((int *)CMSG_DATA(cmsg))[i];
> + if (i == 0)
> + outfd = f;
> + else
> + close(f);
> + }
> + }
> + }
> +
> + b += n;
> + sz -= n;
> + while (sz > 0) {
> + /* short receive */
> + n = recv(fd, b, sz, 0);
> + if (n == -1) {
> + if (errno == EINTR)
> + continue;
> + err(1, "recv");
> + }
> + if (n == 0)
> + errx(1, "recv: unexpected end of file");
> +
> + b += n;
> + sz -= n;
> + }
> +
> + return outfd;
> }
> Index: main.c
> ===================================================================
> RCS file: /cvs/src/usr.sbin/rpki-client/main.c,v
> retrieving revision 1.105
> diff -u -p -r1.105 main.c
> --- main.c 23 Feb 2021 14:25:29 -0000 1.105
> +++ main.c 25 Feb 2021 09:34:00 -0000
> @@ -51,10 +51,12 @@
> /*
> * An rsync repository.
> */
> +#define REPO_MAX_URI 2
> struct repo {
> char *repouri; /* CA repository base URI */
> char *local; /* local path name */
> - char *uris[2]; /* URIs to fetch from */
> + char *temp; /* temporary file / dir */
> + char *uris[REPO_MAX_URI]; /* URIs to fetch from */
> size_t id; /* identifier (array index) */
> int uriidx; /* which URI is fetched */
> int loaded; /* whether loaded or not */
> @@ -91,7 +93,7 @@ RB_HEAD(filepath_tree, filepath);
> RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
>
> static struct filepath_tree fpt = RB_INITIALIZER(&fpt);
> -static struct msgbuf procq, rsyncq;
> +static struct msgbuf procq, rsyncq, httpq;
> static int cachefd;
>
> const char *bird_tablename = "ROAS";
> @@ -101,6 +103,9 @@ int noop;
>
> struct stats stats;
>
> +static void repo_fetch(struct repo *);
> +static char *ta_filename(const struct repo *, int);
> +
> /*
> * Log a message to stderr if and only if "verbose" is non-zero.
> * This uses the err(3) functionality.
> @@ -279,6 +284,70 @@ repo_alloc(void)
> }
>
> static void
> +http_ta_fetch(struct repo *rp)
> +{
> + struct ibuf *b;
> + int filefd;
> +
> + rp->temp = ta_filename(rp, 1);
> +
> + filefd = mkostempat(cachefd, rp->temp, O_CLOEXEC);
> + if (filefd == -1) {
> + err(1, "mkostempat: %s", rp->temp);
> + /* XXX switch to soft fail and restart with next file */
> + }
> + if (fchmod(filefd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1)
> + warn("fchmod: %s", rp->temp);
> +
> + if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL)
> + err(1, NULL);
> + io_simple_buffer(b, &rp->id, sizeof(rp->id));
> + io_str_buffer(b, rp->uris[rp->uriidx]);
> + /* TODO last modified time */
> + io_str_buffer(b, NULL);
> + /* pass file as fd */
> + b->fd = filefd;
> + ibuf_close(&httpq, b);
> +}
> +
> +static int
> +http_done(struct repo *rp, int ok)
> +{
> + if (rp->repouri == NULL) {
> + /* Move downloaded TA file into place, or unlink on failure. */
> + if (ok) {
> + char *file;
> +
> + file = ta_filename(rp, 0);
> + if (renameat(cachefd, rp->temp, cachefd, file) == -1)
> + warn("rename to %s", file);
> + } else {
> + if (unlinkat(cachefd, rp->temp, 0) == -1)
> + warn("unlink %s", rp->temp);
> + }
> + free(rp->temp);
> + rp->temp = NULL;
> + }
> +
> + if (!ok && rp->uriidx < REPO_MAX_URI - 1 &&
> + rp->uris[rp->uriidx + 1] != NULL) {
> + logx("%s: load from network failed, retry", rp->local);
> +
> + rp->uriidx++;
> + repo_fetch(rp);
> + return 0;
> + }
> +
> + if (ok)
> + logx("%s: loaded from network", rp->local);
> + else
> + logx("%s: load from network failed, "
> + "fallback to cache", rp->local);
> +
> + return 1;
> +}
> +
> +static void
> repo_fetch(struct repo *rp)
> {
> struct ibuf *b;
> @@ -293,20 +362,31 @@ repo_fetch(struct repo *rp)
>
> /*
> * Create destination location.
> - * Build up the tree to this point because GPL rsync(1)
> - * will not build the destination for us.
> + * Build up the tree to this point.
> */
>
> if (mkpathat(cachefd, rp->local) == -1)
> err(1, "%s", rp->local);
>
> - logx("%s: pulling from network", rp->local);
> - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL)
> - err(1, NULL);
> - io_simple_buffer(b, &rp->id, sizeof(rp->id));
> - io_str_buffer(b, rp->local);
> - io_str_buffer(b, rp->uris[0]);
> - ibuf_close(&rsyncq, b);
> + logx("%s: pulling from %s", rp->local, rp->uris[rp->uriidx]);
> +
> + if (strncasecmp(rp->uris[rp->uriidx], "rsync://", 8) == 0) {
> + if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL)
> + err(1, NULL);
> + io_simple_buffer(b, &rp->id, sizeof(rp->id));
> + io_str_buffer(b, rp->local);
> + io_str_buffer(b, rp->uris[rp->uriidx]);
> + ibuf_close(&rsyncq, b);
> + } else {
> + /*
> + * Two cases for https. TA files load directly while
> + * for RRDP XML files are downloaded an parsed to build
> + * the repo. TA repos have a NULL repouri.
> + */
> + if (rp->repouri == NULL) {
> + http_ta_fetch(rp);
> + }
> + }
> }
>
> /*
> @@ -333,13 +413,11 @@ ta_lookup(const struct tal *tal)
> rp = repo_alloc();
> rp->local = local;
> for (i = 0, j = 0; i < tal->urisz && j < 2; i++) {
> - if (strncasecmp(tal->uri[i], "rsync://", 8) != 0)
> - continue; /* ignore non rsync URI for now */
> if ((rp->uris[j++] = strdup(tal->uri[i])) == NULL)
> err(1, "strdup");
> }
> if (j == 0)
> - errx(1, "TAL file has no rsync:// URI");
> + errx(1, "TAL %s has no URI", tal->descr);
>
> repo_fetch(rp);
> return rp;
> @@ -379,6 +457,23 @@ repo_lookup(const char *uri)
> return rp;
> }
>
> +static char *
> +ta_filename(const struct repo *repo, int temp)
> +{
> + const char *file;
> + char *nfile;
> +
> + /* does not matter which URI, all end with same filename */
> + file = strrchr(repo->uris[0], '/');
> + assert(file);
> +
> + if (asprintf(&nfile, "%s%s%s", repo->local, file,
> + temp ? ".XXXXXXXX": "") == -1)
> + err(1, "asprintf");
> +
> + return nfile;
> +}
> +
> /*
> * Build local file name base on the URI and the repo info.
> */
> @@ -521,18 +616,13 @@ queue_add_from_tal(struct entityq *q, co
> {
> char *nfile;
> const struct repo *repo;
> - const char *uri;
>
> assert(tal->urisz);
>
> /* Look up the repository. */
> repo = ta_lookup(tal);
>
> - uri = strrchr(repo->uris[0], '/');
> - assert(uri);
> - if (asprintf(&nfile, "%s/%s", repo->local, uri + 1) == -1)
> - err(1, "asprintf");
> -
> + nfile = ta_filename(repo, 0);
> entityq_add(q, nfile, RTYPE_CER, repo, tal->pkey,
> tal->pkeysz, tal->descr);
> }
> @@ -771,13 +861,13 @@ suicide(int sig __attribute__((unused)))
> int
> main(int argc, char *argv[])
> {
> - int rc = 1, c, proc, st, rsync,
> + int rc = 1, c, st, proc, rsync, http,
> fl = SOCK_STREAM | SOCK_CLOEXEC;
> size_t i, outsz = 0, talsz = 0;
> - pid_t procpid, rsyncpid;
> + pid_t procpid, rsyncpid, httppid;
> int fd[2];
> struct entityq q;
> - struct pollfd pfd[2];
> + struct pollfd pfd[3];
> struct roa **out = NULL;
> char *rsync_prog = "openrsync";
> char *bind_addr = NULL;
> @@ -805,7 +895,8 @@ main(int argc, char *argv[])
> cachedir = RPKI_PATH_BASE_DIR;
> outputdir = RPKI_PATH_OUT_DIR;
>
> - if (pledge("stdio rpath wpath cpath fattr proc exec unveil", NULL) ==
> -1)
> + if (pledge("stdio rpath wpath cpath inet fattr dns sendfd recvfd "
> + "proc exec unveil", NULL) == -1)
> err(1, "pledge");
>
> while ((c = getopt(argc, argv, "b:Bcd:e:jnos:t:T:v")) != -1)
> @@ -953,15 +1044,48 @@ main(int argc, char *argv[])
> } else
> rsync = -1;
>
> - assert(rsync != proc);
> + /*
> + * Create a process that will fetch data via https.
> + * With every request the http process receives a file descriptor
> + * where the data should be written to.
> + */
> +
> + if (!noop) {
> + if (socketpair(AF_UNIX, fl, 0, fd) == -1)
> + err(1, "socketpair");
> + if ((httppid = fork()) == -1)
> + err(1, "fork");
> +
> + if (httppid == 0) {
> + close(proc);
> + close(rsync);
> + close(fd[1]);
> +
> + /* change working directory to the cache directory */
> + if (fchdir(cachefd) == -1)
> + err(1, "fchdir");
> +
> + if (pledge("stdio rpath inet dns recvfd", NULL) == -1)
> + err(1, "pledge");
> +
> + proc_http(bind_addr, fd[0]);
> + /* NOTREACHED */
> + }
> +
> + close(fd[0]);
> + http = fd[1];
> + } else
> + http = -1;
>
> - if (pledge("stdio rpath wpath cpath fattr", NULL) == -1)
> + if (pledge("stdio rpath wpath cpath fattr sendfd", NULL) == -1)
> err(1, "pledge");
>
> msgbuf_init(&procq);
> msgbuf_init(&rsyncq);
> + msgbuf_init(&httpq);
> procq.fd = proc;
> rsyncq.fd = rsync;
> + httpq.fd = http;
>
> /*
> * The main process drives the top-down scan to leaf ROAs using
> @@ -971,6 +1095,7 @@ main(int argc, char *argv[])
>
> pfd[0].fd = rsync;
> pfd[1].fd = proc;
> + pfd[2].fd = http;
>
> /*
> * Prime the process with our TAL file.
> @@ -984,22 +1109,27 @@ main(int argc, char *argv[])
> while (entity_queue > 0 && !killme) {
> pfd[0].events = POLLIN;
> if (rsyncq.queued)
> - pfd[0].events = POLLOUT;
> + pfd[0].events |= POLLOUT;
> pfd[1].events = POLLIN;
> if (procq.queued)
> - pfd[1].events = POLLOUT;
> + pfd[1].events |= POLLOUT;
> + pfd[2].events = POLLIN;
> + if (httpq.queued)
> + pfd[2].events |= POLLOUT;
>
> - if ((c = poll(pfd, 2, INFTIM)) == -1) {
> + if ((c = poll(pfd, 3, INFTIM)) == -1) {
> if (errno == EINTR)
> continue;
> err(1, "poll");
> }
>
> if ((pfd[0].revents & (POLLERR|POLLNVAL)) ||
> - (pfd[1].revents & (POLLERR|POLLNVAL)))
> + (pfd[1].revents & (POLLERR|POLLNVAL)) ||
> + (pfd[2].revents & (POLLERR|POLLNVAL)))
> errx(1, "poll: bad fd");
> if ((pfd[0].revents & POLLHUP) ||
> - (pfd[1].revents & POLLHUP))
> + (pfd[1].revents & POLLHUP) ||
> + (pfd[2].revents & POLLHUP))
> errx(1, "poll: hangup");
>
> if (pfd[0].revents & POLLOUT) {
> @@ -1018,9 +1148,18 @@ main(int argc, char *argv[])
> err(1, "write");
> }
> }
> + if (pfd[2].revents & POLLOUT) {
> + switch (msgbuf_write(&httpq)) {
> + case 0:
> + errx(1, "write: connection closed");
> + case -1:
> + err(1, "write");
> + }
> + }
> +
>
> /*
> - * Check the rsync process.
> + * Check the rsync and http process.
> * This means that one of our modules has completed
> * downloading and we can flush the module requests into
> * the parser process.
> @@ -1033,17 +1172,31 @@ main(int argc, char *argv[])
> assert(i < rt.reposz);
>
> assert(!rt.repos[i].loaded);
> - rt.repos[i].loaded = 1;
> if (ok)
> logx("%s: loaded from network",
> rt.repos[i].local);
> else
> logx("%s: load from network failed, "
> "fallback to cache", rt.repos[i].local);
> + rt.repos[i].loaded = 1;
> stats.repos++;
> entityq_flush(&q, &rt.repos[i]);
> }
>
> + if ((pfd[2].revents & POLLIN)) {
> + int ok;
> + io_simple_read(http, &i, sizeof(size_t));
> + io_simple_read(http, &ok, sizeof(ok));
> + assert(i < rt.reposz);
> +
> + assert(!rt.repos[i].loaded);
> + if (http_done(&rt.repos[i], ok)) {
> + rt.repos[i].loaded = 1;
> + stats.repos++;
> + entityq_flush(&q, &rt.repos[i]);
> + }
> + }
> +
> /*
> * The parser has finished something for us.
> * Dequeue these one by one.
> @@ -1072,6 +1225,7 @@ main(int argc, char *argv[])
>
> close(proc);
> close(rsync);
> + close(http);
>
> if (waitpid(procpid, &st, 0) == -1)
> err(1, "waitpid");
> @@ -1086,6 +1240,13 @@ main(int argc, char *argv[])
> warnx("rsync process exited abnormally");
> rc = 1;
> }
> +
> + if (waitpid(httppid, &st, 0) == -1)
> + err(1, "waitpid");
> + if (!WIFEXITED(st) || WEXITSTATUS(st) != 0) {
> + warnx("http process exited abnormally");
> + rc = 1;
> + }
> }
> gettimeofday(&now_time, NULL);
> timersub(&now_time, &start_time, &stats.elapsed_time);
> @@ -1120,6 +1281,7 @@ main(int argc, char *argv[])
> for (i = 0; i < rt.reposz; i++) {
> free(rt.repos[i].repouri);
> free(rt.repos[i].local);
> + free(rt.repos[i].temp);
> free(rt.repos[i].uris[0]);
> free(rt.repos[i].uris[1]);
> }
> Index: mkostempat.c
> ===================================================================
> RCS file: mkostempat.c
> diff -N mkostempat.c
> --- /dev/null 1 Jan 1970 00:00:00 -0000
> +++ mkostempat.c 19 Feb 2021 14:11:28 -0000
> @@ -0,0 +1,94 @@
> +/* $OpenBSD$ */
> +/*
> + * Copyright (c) 1996-1998, 2008 Theo de Raadt
> + * Copyright (c) 1997, 2008-2009 Todd C. Miller
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <errno.h>
> +#include <fcntl.h>
> +#include <limits.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <string.h>
> +#include <ctype.h>
> +#include <unistd.h>
> +
> +#include "extern.h"
> +
> +#define TEMPCHARS
> "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
> +#define NUM_CHARS (sizeof(TEMPCHARS) - 1)
> +#define MIN_X 6
> +
> +#define MKOTEMP_FLAGS (O_APPEND | O_CLOEXEC | O_DSYNC | O_RSYNC |
> O_SYNC)
> +
> +#ifndef nitems
> +#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0]))
> +#endif
> +
> +int
> +mkostempat(int dfd, char *path, int flags)
> +{
> + char *start, *cp, *ep;
> + const char tempchars[] = TEMPCHARS;
> + unsigned int tries;
> + size_t len;
> + int fd;
> +
> + len = strlen(path);
> + if (len < MIN_X) {
> + errno = EINVAL;
> + return(-1);
> + }
> + ep = path + len;
> +
> + for (start = ep; start > path && start[-1] == 'X'; start--)
> + ;
> + if (ep - start < MIN_X) {
> + errno = EINVAL;
> + return(-1);
> + }
> +
> + if (flags & ~MKOTEMP_FLAGS) {
> + errno = EINVAL;
> + return(-1);
> + }
> + flags |= O_CREAT | O_EXCL | O_RDWR;
> +
> + tries = INT_MAX;
> + do {
> + cp = start;
> + do {
> + unsigned short rbuf[16];
> + unsigned int i;
> +
> + /*
> + * Avoid lots of arc4random() calls by using
> + * a buffer sized for up to 16 Xs at a time.
> + */
> + arc4random_buf(rbuf, sizeof(rbuf));
> + for (i = 0; i < nitems(rbuf) && cp != ep; i++)
> + *cp++ = tempchars[rbuf[i] % NUM_CHARS];
> + } while (cp != ep);
> +
> + fd = openat(dfd, path, flags, S_IRUSR|S_IWUSR);
> + if (fd != -1 || errno != EEXIST)
> + return(fd);
> + } while (--tries);
> +
> + errno = EEXIST;
> + return(-1);
> +}
>
