I wanted to benchmark a http-connection using fetch in a
shell-script. I came to realize that fetch doesn't support
compressed streams when I force-fed it the compressed stream by
ignoring the absence of Accept-Encoding. This doesn't influence my
benchmarking, but I think fetch should have this feature.
I hacked together deflate support for the http part of libfetch.
Next on my todo list is proper error handling, gzip support, code
clean up and general code clean up in http.c (in order of priority).
I'd love to get some feedback, do you consider this useful? Does it
work on your system? Would there be a chance of getting the
finalized version into SVN?
The attached patch applies to RELENG_7. Probably everywhere else,
too. Because I think libfetch development has been at a halt for
some time.
Regards,
Dominic
--- src/lib/libfetch/http.c.orig 2008-06-29 15:28:58.000000000 +0200
+++ src/lib/libfetch/http.c 2008-06-30 19:38:57.000000000 +0200
@@ -75,6 +75,7 @@
#include <string.h>
#include <time.h>
#include <unistd.h>
+#include <zlib.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
@@ -105,7 +106,6 @@
#define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599)
-
/*****************************************************************************
* I/O functions for decoding chunked streams
*/
@@ -126,6 +126,16 @@
#endif
};
+struct zlibio
+{
+ FILE *source; /* the http connection to read from */
+ z_stream *stream; /* the zlib stream to read the */
+ /* uncompressed data from */
+ char in[65536]; /* read buffer */
+};
+
+typedef FILE * (*funopen_function)(conn_t *, int);
+
/*
* Get next chunk header
*/
@@ -302,10 +312,50 @@
}
/*
+ * Read function for deflate compressed data.
+ */
+static int
+http_inflate_readfn(void *v, char *buf, int len)
+{
+ struct zlibio *io = (struct zlibio *)v;
+ int status;
+
+ /* Only read if the last read chunk has completely been flushed. */
+ if (io->stream->avail_in == 0) {
+ io->stream->avail_in = fread(io->in, sizeof(char),
sizeof(io->in), io->source);
+
+ /* Forward errors and eof */
+ if (io->stream->avail_in <= 0)
+ return io->stream->avail_in;
+
+ io->stream->next_in = io->in;
+ }
+
+ io->stream->avail_out = len;
+ io->stream->next_out = buf;
+ status = inflate(io->stream, Z_SYNC_FLUSH);
+
+ return (len - io->stream->avail_out);
+}
+
+/*
+ * Close function for deflate compressed data
+ */
+static int
+http_inflate_closefn(void *v)
+{
+ struct zlibio *io = (struct zlibio *)v;
+
+ (void)inflateEnd(io->stream);
+ free(io->stream);
+ return (fclose(io->source));
+}
+
+/*
* Wrap a file descriptor up
*/
static FILE *
-http_funopen(conn_t *conn, int chunked)
+http_funopen_raw(conn_t *conn, int chunked)
{
struct httpio *io;
FILE *f;
@@ -316,7 +366,7 @@
}
io->conn = conn;
io->chunked = chunked;
- f = funopen(io, http_readfn, http_writefn, NULL, http_closefn);
+ f = funopen(io, &http_readfn, &http_writefn, NULL, &http_closefn);
if (f == NULL) {
fetch_syserr();
free(io);
@@ -325,6 +375,50 @@
return (f);
}
+/*
+ * Wrap a file descriptor up around the zlip inflate command
+ */
+static FILE *
+http_funopen_inflate(conn_t *conn, int chunked)
+{
+ struct zlibio *io;
+ FILE *f;
+
+ if ((io = calloc(1, sizeof(*io))) == NULL) {
+ fetch_syserr();
+ return (NULL);
+ }
+
+ io->source = http_funopen_raw(conn, chunked);
+
+ if ((io->stream = calloc(1, sizeof(*(io->stream)))) == NULL) {
+ fetch_syserr();
+ free(io);
+ return (NULL);
+ }
+
+ io->stream->zalloc = Z_NULL;
+ io->stream->zfree = Z_NULL;
+ io->stream->opaque = Z_NULL;
+ io->stream->avail_in = 0;
+ io->stream->next_in = Z_NULL;
+ if (inflateInit2(io->stream, -MAX_WBITS) != Z_OK) {
+ fetch_syserr();
+ free(io->source);
+ free(io);
+ return (NULL);
+ }
+
+ f = funopen(io, &http_inflate_readfn, NULL, NULL,
&http_inflate_closefn);
+
+ if (f == NULL) {
+ fetch_syserr();
+ free(io->source);
+ free(io);
+ return (NULL);
+ }
+ return f;
+}
/*****************************************************************************
* Helper functions for talking to the server and parsing its replies
@@ -336,6 +430,7 @@
hdr_error = -1,
hdr_end = 0,
hdr_unknown = 1,
+ hdr_content_encoding,
hdr_content_length,
hdr_content_range,
hdr_last_modified,
@@ -349,6 +444,7 @@
hdr_t num;
const char *name;
} hdr_names[] = {
+ { hdr_content_encoding, "Content-Encoding" },
{ hdr_content_length, "Content-Length" },
{ hdr_content_range, "Content-Range" },
{ hdr_last_modified, "Last-Modified" },
@@ -496,6 +592,24 @@
}
/*
+ * Parse a content-encoding header
+ */
+funopen_function
+http_parse_encoding(const char *p, off_t *length)
+{
+ off_t len = sizeof("gzip");
+ /* not yet supported
+ if (strncmp(p, "gzip", len) == 0)
+ return &http _funopen_gunzip; */
+
+ len = sizeof("deflate");
+ if (strncmp(p, "deflate", len) == 0)
+ return &http_funopen_inflate;
+
+ return &http_funopen_raw;
+}
+
+/*
* Parse a content-length header
*/
static int
@@ -800,6 +914,7 @@
conn_t *conn;
struct url *url, *new;
int chunked, direct, need_auth, noredirect, verbose;
+ funopen_function funopenfn;
int e, i, n, val;
off_t offset, clength, length, size;
time_t mtime;
@@ -834,6 +949,7 @@
length = -1;
size = -1;
mtime = 0;
+ funopenfn = &http_funopen_raw;
/* check port */
if (!url->port)
@@ -919,6 +1035,7 @@
http_cmd(conn, "User-Agent: %s " _LIBFETCH_VER,
getprogname());
if (url->offset > 0)
http_cmd(conn, "Range: bytes=%lld-", (long
long)url->offset);
+ http_cmd(conn, "Accept-Encoding: deflate");
http_cmd(conn, "Connection: close");
http_cmd(conn, "");
@@ -999,6 +1116,9 @@
case hdr_error:
http_seterr(HTTP_PROTOCOL_ERROR);
goto ouch;
+ case hdr_content_encoding:
+ funopenfn = http_parse_encoding(p, &length);
+ break;
case hdr_content_length:
http_parse_length(p, &clength);
break;
@@ -1119,7 +1239,9 @@
/* fill in stats */
if (us) {
- us->size = size;
+ /* We only know the real output size for uncompressed data. */
+ if (funopenfn == &http_funopen_raw)
+ us->size = size;
us->atime = us->mtime = mtime;
}
@@ -1134,7 +1256,7 @@
URL->length = clength;
/* wrap it up in a FILE */
- if ((f = http_funopen(conn, chunked)) == NULL) {
+ if ((f = (*funopenfn)(conn, chunked)) == NULL) {
fetch_syserr();
goto ouch;
}
--- src/lib/libfetch/Makefile.orig 2008-06-29 23:45:32.000000000 +0200
+++ src/lib/libfetch/Makefile 2008-06-29 23:43:11.000000000 +0200
@@ -20,6 +20,8 @@
LDADD= -lssl -lcrypto
.endif
+LDADD+= -lz
+
CFLAGS+= -DFTP_COMBINE_CWDS -l
CSTD?= c99
_______________________________________________
freebsd-hackers@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-hackers
To unsubscribe, send any mail to "[EMAIL PROTECTED]"
