Patch V2: Fix minor issues found by Steffan
Patch V3: split wire codes and compression flags
Patch V4: Fix further issues reported by Gert
Patch V5: really fix the issues that should be fixed in v2
It has been tested against v3 server and again itself. From James Mail:
Compression V2
I have observed that compression in many cases, even when
enabled, often does not produce packet size reduction
because much of the packet data typically generated by web
sessions is already compressed. Further, the single byte that
precedes the packet and indicates whether or not compression
occurred has the unfortunate side effect of misaligning the IP
packet in cases where compression did not occur. To remedy this,
I propose a Compression V2 header that is optimized for the
case where compression does not occur.
a. No compression occurred and first byte of IP/Ethernet packet
is NOT 0x50 (0 bytes of overhead and maintains alignment):
[ uncompressed IP/Ethernet packet ]
b. No compression occurred and first byte of IP/Ethernet packet
is 0x50 (2 bytes of overhead but unlikely since no known
IP packet can begin with 0x50):
[ 0x50 ] [ 0x00 ] [ uncompressed IP/Ethernet packet ]
c. Compression occurred (2 bytes of overhead):
[ 0x50 ] [ compression Alg ID ] [ compressed IP/Ethernet packet ]
Compression Alg ID is one-byte algorithm identifier
for LZ4 (0x1), LZO (0x2), or Snappy (0x3).
This approach has several beneficial effects:
1. In the common case where compression does not occur, no
compression op is required, therefore there is zero overhead.
2. When compression does not occur, the IP/Ethernet packet
alignment is retained.
3. This technique does not require any byte swapping with
the tail of the packet which can potentially incur an
expensive cache miss.
---
src/openvpn/comp-lz4.c | 199 ++++++++++++++++++++++++++++++++++++++-----------
src/openvpn/comp-lz4.h | 1 +
src/openvpn/comp.c | 31 ++++++++
src/openvpn/comp.h | 25 +++++++
src/openvpn/compstub.c | 51 +++++++++++++
src/openvpn/lzo.c | 3 -
src/openvpn/options.c | 10 ++-
7 files changed, 274 insertions(+), 46 deletions(-)
diff --git a/src/openvpn/comp-lz4.c b/src/openvpn/comp-lz4.c
index 4651148..ddd8e28 100644
--- a/src/openvpn/comp-lz4.c
+++ b/src/openvpn/comp-lz4.c
@@ -44,9 +44,6 @@
#include "memdbg.h"
-/* Initial command byte to tell our peer if we compressed */
-#define LZ4_COMPRESS_BYTE 0x69
-
static void
lz4_compress_init (struct compress_context *compctx)
{
@@ -55,20 +52,22 @@ lz4_compress_init (struct compress_context *compctx)
}
static void
-lz4_compress_uninit (struct compress_context *compctx)
+lz4v2_compress_init (struct compress_context *compctx)
{
+ msg (D_INIT_MEDIUM, "LZ4v2 compression initializing");
}
static void
-lz4_compress (struct buffer *buf, struct buffer work,
- struct compress_context *compctx,
- const struct frame* frame)
+lz4_compress_uninit (struct compress_context *compctx)
{
- bool compressed = false;
-
- if (buf->len <= 0)
- return;
+}
+static bool
+do_lz4_compress (struct buffer *buf,
+ struct buffer *work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
/*
* In order to attempt compression, length must be at least
COMPRESS_THRESHOLD.
*/
@@ -78,38 +77,57 @@ lz4_compress (struct buffer *buf, struct buffer work,
int zlen_max = ps + COMP_EXTRA_BUFFER (ps);
int zlen;
- ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
- ASSERT (buf_safe (&work, zlen_max));
+ ASSERT (buf_init (work, FRAME_HEADROOM (frame)));
+ ASSERT (buf_safe (work, zlen_max));
if (buf->len > ps)
{
dmsg (D_COMP_ERRORS, "LZ4 compression buffer overflow");
buf->len = 0;
- return;
+ return false;
}
- zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char
*)BPTR(&work), BLEN(buf), zlen_max );
+ zlen = LZ4_compress_limitedOutput((const char *)BPTR(buf), (char
*)BPTR(work), BLEN(buf), zlen_max );
if (zlen <= 0)
{
dmsg (D_COMP_ERRORS, "LZ4 compression error");
buf->len = 0;
- return;
+ return false;
}
- ASSERT (buf_safe (&work, zlen));
- work.len = zlen;
- compressed = true;
+ ASSERT (buf_safe (work, zlen));
+ work->len = zlen;
+
- dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work.len);
+ dmsg (D_COMP, "LZ4 compress %d -> %d", buf->len, work->len);
compctx->pre_compress += buf->len;
- compctx->post_compress += work.len;
+ compctx->post_compress += work->len;
+ return true;
}
+ return false;
+}
+
+
+static void
+lz4_compress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ bool compressed;
+ if (buf->len <= 0)
+ return;
+
+ compressed = do_lz4_compress(buf, &work, compctx, frame);
+
+ /* On error do_lz4_compress sets buf len to zero, just return */
+ if (buf->len == 0)
+ return;
/* did compression save us anything? */
{
uint8_t comp_head_byte = NO_COMPRESS_BYTE_SWAP;
- if (compressed && work.len < buf->len)
+ if (compressed && work.len +2 < buf->len)
{
*buf = work;
comp_head_byte = LZ4_COMPRESS_BYTE;
@@ -128,13 +146,72 @@ lz4_compress (struct buffer *buf, struct buffer work,
}
}
+
+static void
+lz4v2_compress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ bool compressed;
+ if (buf->len <= 0)
+ return;
+
+ compressed = do_lz4_compress(buf, &work, compctx, frame);
+
+ /* On Error just return */
+ if (buf->len == 0)
+ return;
+
+ /* did compression save us anything? Include 2 byte compression header
+ in calculation */
+ {
+ if (compressed && work.len + 2 < buf->len)
+ {
+ ASSERT(buf_prepend(&work, 2));
+ uint8_t *head = BPTR (&work);
+ head[0] = COMP_ALGV2_INDICATOR_BYTE;
+ head[1] = COMP_ALGV2_LZ4_BYTE;
+ *buf = work;
+ }
+ else
+ {
+ compv2_escape_data_ifneeded(buf);
+ }
+ }
+}
+
+void
+do_lz4_decompress(size_t zlen_max,
+ struct buffer *work,
+ struct buffer *buf,
+ struct compress_context *compctx)
+{
+ int uncomp_len;
+ ASSERT (buf_safe (work, zlen_max));
+ uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char
*)BPTR(work), (size_t)BLEN(buf), zlen_max);
+ if (uncomp_len <= 0)
+ {
+ dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
+ buf->len = 0;
+ return;
+ }
+
+ ASSERT (buf_safe (work, uncomp_len));
+ work->len = uncomp_len;
+
+ dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work->len);
+ compctx->pre_decompress += buf->len;
+ compctx->post_decompress += work->len;
+
+ *buf = *work;
+}
+
static void
lz4_decompress (struct buffer *buf, struct buffer work,
- struct compress_context *compctx,
- const struct frame* frame)
+ struct compress_context *compctx,
+ const struct frame* frame)
{
size_t zlen_max = EXPANDED_SIZE (frame);
- int uncomp_len;
uint8_t c; /* flag indicating whether or not our peer compressed */
if (buf->len <= 0)
@@ -152,23 +229,7 @@ lz4_decompress (struct buffer *buf, struct buffer work,
if (c == LZ4_COMPRESS_BYTE) /* packet was compressed */
{
- ASSERT (buf_safe (&work, zlen_max));
- uncomp_len = LZ4_decompress_safe((const char *)BPTR(buf), (char
*)BPTR(&work), (size_t)BLEN(buf), zlen_max);
- if (uncomp_len <= 0)
- {
- dmsg (D_COMP_ERRORS, "LZ4 decompression error: %d", uncomp_len);
- buf->len = 0;
- return;
- }
-
- ASSERT (buf_safe (&work, uncomp_len));
- work.len = uncomp_len;
-
- dmsg (D_COMP, "LZ4 decompress %d -> %d", buf->len, work.len);
- compctx->pre_decompress += buf->len;
- compctx->post_decompress += work.len;
-
- *buf = work;
+ do_lz4_decompress(zlen_max, &work, buf, compctx);
}
else if (c == NO_COMPRESS_BYTE_SWAP) /* packet was not compressed */
{
@@ -181,6 +242,52 @@ lz4_decompress (struct buffer *buf, struct buffer work,
}
}
+static void
+lz4v2_decompress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ size_t zlen_max = EXPANDED_SIZE (frame);
+ uint8_t c; /* flag indicating whether or not our peer compressed */
+
+ if (buf->len <= 0)
+ return;
+
+ ASSERT (buf_init (&work, FRAME_HEADROOM (frame)));
+
+ /* do unframing/swap (assumes buf->len > 0) */
+ uint8_t *head = BPTR (buf);
+ c = *head;
+
+ /* Not compressed */
+ if (c != COMP_ALGV2_INDICATOR_BYTE) {
+ return;
+ }
+
+ /* Packet to short to make sense */
+ if (buf->len <= 1)
+ {
+ buf->len=0;
+ return;
+ }
+
+ c = head[1];
+ if (c == COMP_ALGV2_LZ4_BYTE) /* packet was compressed */
+ {
+ buf_advance(buf,2);
+ do_lz4_decompress(zlen_max, &work, buf, compctx);
+ }
+ else if (c == COMP_ALGV2_UNCOMPRESSED_BYTE)
+ {
+ buf_advance(buf,2);
+ }
+ else
+ {
+ dmsg (D_COMP_ERRORS, "Bad LZ4v2 decompression header byte: %d", c);
+ buf->len = 0;
+ }
+}
+
const struct compress_alg lz4_alg = {
"lz4",
lz4_compress_init,
@@ -189,6 +296,14 @@ const struct compress_alg lz4_alg = {
lz4_decompress
};
+const struct compress_alg lz4v2_alg = {
+ "lz4v2",
+ lz4v2_compress_init,
+ lz4_compress_uninit,
+ lz4v2_compress,
+ lz4v2_decompress
+};
+
#else
static void dummy(void) {}
#endif /* ENABLE_LZ4 */
diff --git a/src/openvpn/comp-lz4.h b/src/openvpn/comp-lz4.h
index ca1dfa9..9d3c664 100644
--- a/src/openvpn/comp-lz4.h
+++ b/src/openvpn/comp-lz4.h
@@ -31,6 +31,7 @@
#include "buffer.h"
extern const struct compress_alg lz4_alg;
+extern const struct compress_alg lz4v2_alg;
struct lz4_workspace
{
diff --git a/src/openvpn/comp.c b/src/openvpn/comp.c
index 706ad7e..809c83d 100644
--- a/src/openvpn/comp.c
+++ b/src/openvpn/comp.c
@@ -50,6 +50,11 @@ comp_init(const struct compress_options *opt)
compctx->alg = comp_stub_alg;
(*compctx->alg.compress_init)(compctx);
break;
+ case COMP_ALGV2_UNCOMPRESSED:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = compv2_stub_alg;
+ break;
#ifdef ENABLE_LZO
case COMP_ALG_LZO:
ALLOC_OBJ_CLEAR (compctx, struct compress_context);
@@ -65,11 +70,35 @@ comp_init(const struct compress_options *opt)
compctx->alg = lz4_alg;
(*compctx->alg.compress_init)(compctx);
break;
+ case COMP_ALGV2_LZ4:
+ ALLOC_OBJ_CLEAR (compctx, struct compress_context);
+ compctx->flags = opt->flags;
+ compctx->alg = lz4v2_alg;
+ break;
#endif
}
return compctx;
}
+/* In the v2 compression schmemes, an uncompressed packet has
+ * has no opcode in front, unless the first byte is 0x50. In this
+ * case the packet needs to be escaped */
+void
+compv2_escape_data_ifneeded (struct buffer *buf)
+{
+ uint8_t *head = BPTR (buf);
+ if (head[0] != COMP_ALGV2_INDICATOR_BYTE)
+ return;
+
+ /* Header is 0x50 */
+ ASSERT(buf_prepend(buf, 2));
+
+ head = BPTR (buf);
+ head[0] = COMP_ALGV2_INDICATOR_BYTE;
+ head[1] = COMP_ALGV2_UNCOMPRESSED;
+}
+
+
void
comp_uninit(struct compress_context *compctx)
{
@@ -120,6 +149,7 @@ comp_generate_peer_info_string(const struct
compress_options *opt, struct buffer
{
#if defined(ENABLE_LZ4)
buf_printf (out, "IV_LZ4=1\n");
+ buf_printf (out, "IV_LZ4v2=1\n");
#endif
#if defined(ENABLE_LZO)
buf_printf (out, "IV_LZO=1\n");
@@ -129,6 +159,7 @@ comp_generate_peer_info_string(const struct
compress_options *opt, struct buffer
if (!lzo_avail)
buf_printf (out, "IV_LZO_STUB=1\n");
buf_printf (out, "IV_COMP_STUB=1\n");
+ buf_printf (out, "IV_COMP_STUBv2=1\n");
}
}
diff --git a/src/openvpn/comp.h b/src/openvpn/comp.h
index 716b1c0..9ed9532 100644
--- a/src/openvpn/comp.h
+++ b/src/openvpn/comp.h
@@ -43,12 +43,22 @@
#define COMP_ALG_SNAPPY 3 /* Snappy algorithm (no longer supported) */
#define COMP_ALG_LZ4 4 /* LZ4 algorithm */
+
+/* algorithm v2 */
+#define COMP_ALGV2_UNCOMPRESSED 10
+#define COMP_ALGV2_LZ4 11
+/*
+#define COMP_ALGV2_LZO 12
+#define COMP_ALGV2_SNAPPY 13
+*/
+
/* Compression flags */
#define COMP_F_ADAPTIVE (1<<0) /* COMP_ALG_LZO only */
#define COMP_F_ASYM (1<<1) /* only downlink is compressed, not uplink */
#define COMP_F_SWAP (1<<2) /* initial command byte is swapped with last
byte in buffer to preserve payload alignment */
#define COMP_F_ADVERTISE_STUBS_ONLY (1<<3) /* tell server that we only support
compression stubs */
+
/*
* Length of prepended prefix on compressed packets
*/
@@ -57,9 +67,21 @@
/*
* Prefix bytes
*/
+
+/* V1 on wire codes */
+/* Initial command byte to tell our peer if we compressed */
+#define LZO_COMPRESS_BYTE 0x66
+#define LZ4_COMPRESS_BYTE 0x69
#define NO_COMPRESS_BYTE 0xFA
#define NO_COMPRESS_BYTE_SWAP 0xFB /* to maintain payload alignment, replace
this byte with last byte of packet */
+/* V2 on wire code */
+#define COMP_ALGV2_INDICATOR_BYTE 0x50
+#define COMP_ALGV2_UNCOMPRESSED_BYTE 0
+#define COMP_ALGV2_LZ4_BYTE 1
+#define COMP_ALGV2_LZO_BYTE 2
+#define COMP_ALGV2_SNAPPY_BYTE 3
+
/*
* Compress worst case size expansion (for any algorithm)
*
@@ -145,6 +167,7 @@ struct compress_context
};
extern const struct compress_alg comp_stub_alg;
+extern const struct compress_alg compv2_stub_alg;
struct compress_context *comp_init(const struct compress_options *opt);
@@ -157,6 +180,8 @@ void comp_print_stats (const struct compress_context
*compctx, struct status_out
void comp_generate_peer_info_string(const struct compress_options *opt, struct
buffer *out);
+void compv2_escape_data_ifneeded (struct buffer *buf);
+
static inline bool
comp_enabled(const struct compress_options *info)
{
diff --git a/src/openvpn/compstub.c b/src/openvpn/compstub.c
index 2ab7163..9c6aad2 100644
--- a/src/openvpn/compstub.c
+++ b/src/openvpn/compstub.c
@@ -105,6 +105,57 @@ stub_decompress (struct buffer *buf, struct buffer work,
}
}
+
+static void
+stubv2_compress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ if (buf->len <= 0)
+ return;
+
+ compv2_escape_data_ifneeded (buf);
+}
+
+static void
+stubv2_decompress (struct buffer *buf, struct buffer work,
+ struct compress_context *compctx,
+ const struct frame* frame)
+{
+ if (buf->len <= 0)
+ return;
+
+ uint8_t *head = BPTR (buf);
+
+ /* no compression or packet to short*/
+ if (head[0] != COMP_ALGV2_INDICATOR_BYTE)
+ return;
+
+ /* compression header (0x50) is present */
+ buf_advance(buf, 1);
+
+ /* Packet buffer too short (only 1 byte) */
+ if (buf->len <= 0)
+ return;
+
+ head = BPTR (buf);
+ buf_advance(buf, 1);
+
+ if (head[0] != COMP_ALGV2_UNCOMPRESSED_BYTE) {
+ dmsg (D_COMP_ERRORS, "Bad compression stubv2 decompression header byte:
%d", *head);
+ buf->len = 0;
+ return;
+ }
+}
+
+const struct compress_alg compv2_stub_alg = {
+ "stubv2",
+ stub_compress_init,
+ stub_compress_uninit,
+ stubv2_compress,
+ stubv2_decompress
+};
+
const struct compress_alg comp_stub_alg = {
"stub",
stub_compress_init,
diff --git a/src/openvpn/lzo.c b/src/openvpn/lzo.c
index daa02ed..25a839b 100644
--- a/src/openvpn/lzo.c
+++ b/src/openvpn/lzo.c
@@ -42,9 +42,6 @@
#include "memdbg.h"
-/* Initial command byte to tell our peer if we compressed */
-#define LZO_COMPRESS_BYTE 0x66
-
/**
* Perform adaptive compression housekeeping.
*
diff --git a/src/openvpn/options.c b/src/openvpn/options.c
index 5a83897..b00059a 100644
--- a/src/openvpn/options.c
+++ b/src/openvpn/options.c
@@ -6318,16 +6318,20 @@ add_option (struct options *options,
VERIFY_PERMISSION (OPT_P_COMP);
if (p[1])
{
+ options->comp.flags = 0;
if (streq (p[1], "stub"))
{
options->comp.alg = COMP_ALG_STUB;
options->comp.flags = (COMP_F_SWAP|COMP_F_ADVERTISE_STUBS_ONLY);
}
+ else if (streq(p[1], "stub-v2"))
+ {
+ options->comp.alg = COMP_ALGV2_UNCOMPRESSED;
+ }
#if defined(ENABLE_LZO)
else if (streq (p[1], "lzo"))
{
options->comp.alg = COMP_ALG_LZO;
- options->comp.flags = 0;
}
#endif
#if defined(ENABLE_LZ4)
@@ -6336,6 +6340,10 @@ add_option (struct options *options,
options->comp.alg = COMP_ALG_LZ4;
options->comp.flags = COMP_F_SWAP;
}
+ else if (streq (p[1], "lz4-v2"))
+ {
+ options->comp.alg = COMP_ALGV2_LZ4;
+ }
#endif
else
{
--
2.5.4 (Apple Git-61)
