diff --git a/src/common/pg_lzcompress.c b/src/common/pg_lzcompress.c index c8e9ef9..7534d07 100644 --- a/src/common/pg_lzcompress.c +++ b/src/common/pg_lzcompress.c @@ -107,6 +107,23 @@ * and end up with a total compression rate of 96%, what's still * worth a Whow. * + * Version 2 streams start with a marker that is an invalid legacy + * offset-zero match. The marker is followed by a 256-bit literal + * alphabet bitmap and packed canonical Huffman code lengths. The + * remaining bit stream uses one bit to distinguish literals from + * matches. Literals use their Huffman code. Matches use an 8- or + * 16-bit offset and a compact variable-length match length, allowing + * a 64KB history window and matches up to 65535 bytes. New decoders + * continue to accept the legacy representation described above. + * + * Version 3 uses the same LZ77 parser and limits, but stores literals + * as byte-aligned runs. A one-byte sequence header contains four-bit + * literal and match lengths, extended with ULEB128 when necessary. + * Each match has a fixed two-byte offset. This representation gives + * up some literal entropy coding in exchange for a much simpler decode + * loop. If it cannot meet the requested compression rate, the encoder + * uses version 2 instead. + * * The compression algorithm * * The following uses numbers used in the default strategy. @@ -185,7 +202,18 @@ #include +#ifdef USE_AVX2_WITH_RUNTIME_CHECK +#include +#endif + #include "common/pg_lzcompress.h" +#include "port/pg_bswap.h" +#include "port/pg_bitutils.h" +#include "port/pg_cpu.h" +#include "port/simd.h" +#ifndef FRONTEND +#include "utils/memutils.h" +#endif /* ---------- @@ -195,6 +223,112 @@ #define PGLZ_MAX_HISTORY_LISTS 8192 /* must be power of 2 */ #define PGLZ_HISTORY_SIZE 4096 #define PGLZ_MAX_MATCH 273 +#define PGLZ_FAST_MATCH_PROBES 4 +#define PGLZ_FULL_MATCH_PROBES 8 +#define PGLZ_PROBE_MIN_INPUT (PGLZ_HISTORY_SIZE * 8) +#define PGLZ_PROBE_SAMPLE_SIZE 64 + +/* + * Versions 2 and 3 use a fast single-candidate LZ77 parser with a 64KB window. + * Their magic starts with a legacy control byte and an offset-zero match, + * which can never begin a valid legacy stream, so the decompressor can + * distinguish the formats without ambiguity. + */ +#define PGLZ_V2_MIN_INPUT (PGLZ_HISTORY_SIZE * 8) +#define PGLZ_V2_HASH_BITS 14 +#define PGLZ_V2_HASH_SIZE (1 << PGLZ_V2_HASH_BITS) +#define PGLZ_V2_WINDOW_SIZE 65535 +#define PGLZ_V2_MIN_MATCH 6 +#define PGLZ_V2_MAX_MATCH 65535 +#define PGLZ_V2_HUFF_BITS 13 +#define PGLZ_V2_DECODE_BITS 11 +#define PGLZ_V2_DECODE_EXTRA (PGLZ_V2_HUFF_BITS - PGLZ_V2_DECODE_BITS) +#define PGLZ_V2_SUBTABLE_SIZE (1 << PGLZ_V2_DECODE_EXTRA) +#define PGLZ_V2_MAX_SUBTABLES 256 +#define PGLZ_V2_SUBTABLE_FLAG 0x80000000U +#define PGLZ_V2_LITERAL_PAIR_FLAG 0x40000000U +#define PGLZ_V2_PAIR_SYMBOL_SHIFT 16 +#define PGLZ_V2_PAIR_LENGTH_SHIFT 24 +#define PGLZ_V2_BITMAP_SIZE 32 +#define PGLZ_V2_MAX_HEADER (7 + PGLZ_V2_BITMAP_SIZE + 128) + +/* + * Version 3 groups literals into byte-aligned runs. The sequence header is: + * + * 7..4 literal length, with 15 followed by a ULEB128 extension + * 3..0 match length minus PGLZ_V2_MIN_MATCH, extended at 15 + * + * Every sequence except the last literal-only one is followed by a two-byte + * match offset. Fixed-width offsets and byte-aligned literals keep the hot + * decode path branch-light, while the two four-bit lengths make extensions + * uncommon. + */ +#define PGLZ_V3_LITERAL_SHIFT 4 +#define PGLZ_V3_LITERAL_MASK 0x0f +#define PGLZ_V3_MATCH_MASK 0x0f +#define PGLZ_V3_MAX_HEADER (7 + 6) +#define PGLZ_V3_NO_MATCH (-2) + +static const unsigned char pglz_v2_magic[] = { + 0x01, 0x00, 0x00, 'P', 'G', 'L', '2' +}; + +static const unsigned char pglz_v3_magic[] = { + 0x01, 0x00, 0x00, 'P', 'G', 'L', '3' +}; + +#ifndef FRONTEND +#define PGLZ_ALLOC(size) palloc(size) +#define PGLZ_FREE(ptr) pfree(ptr) +#else +#define PGLZ_ALLOC(size) malloc(size) +#define PGLZ_FREE(ptr) free(ptr) +#endif + +typedef struct PGLZ_BitWriter +{ + unsigned char *ptr; + unsigned char *end; + uint64 bits; + int nbits; + bool failed; +} PGLZ_BitWriter; + +typedef struct PGLZ_BitReader +{ + const unsigned char *ptr; + const unsigned char *end; + uint64 bits; + int nbits; +} PGLZ_BitReader; + +typedef struct PGLZ_HuffNode +{ + uint64 frequency; + int16 parent; + uint16 min_symbol; +} PGLZ_HuffNode; + +typedef struct PGLZ_TokenWriter +{ + unsigned char *ptr; + unsigned char *end; + unsigned char *control; + unsigned char mask; + int items_left; +} PGLZ_TokenWriter; + +typedef struct PGLZ_TokenReader +{ + const unsigned char *ptr; + const unsigned char *end; + unsigned char control; + int items_left; +} PGLZ_TokenReader; + +static int32 pglz_decompress_legacy(const char *source, int32 slen, + char *dest, int32 rawsize, + bool check_complete); /* ---------- @@ -202,17 +336,16 @@ * * Linked list for the backward history lookup * - * All the entries sharing a hash key are linked in a doubly linked list. - * This makes it easy to remove an entry when it's time to recycle it - * (because it's more than 4K positions old). + * All the entries sharing a hash key are linked from newest to oldest. Links + * are indexes into hist_entries rather than pointers. Entries live in a + * fixed-size ring, and overwritten links are recognized during lookup from + * their hash key and ring distance. * ---------- */ typedef struct PGLZ_HistEntry { - struct PGLZ_HistEntry *next; /* links for my hash key's list */ - struct PGLZ_HistEntry *prev; - int hindex; /* my current hash key */ - const char *pos; /* my input position */ + uint16 next; /* links for my hash key's list */ + uint16 hindex; /* my current hash key */ } PGLZ_HistEntry; @@ -252,15 +385,13 @@ const PGLZ_Strategy *const PGLZ_strategy_always = &strategy_always_data; * Statically allocated work arrays for history * ---------- */ -static int16 hist_start[PGLZ_MAX_HISTORY_LISTS]; +static uint16 hist_start[PGLZ_MAX_HISTORY_LISTS]; static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; /* - * Element 0 in hist_entries is unused, and means 'invalid'. Likewise, - * INVALID_ENTRY_PTR in next/prev pointers mean 'invalid'. + * Element 0 in hist_entries is unused, and means 'invalid'. */ #define INVALID_ENTRY 0 -#define INVALID_ENTRY_PTR (&hist_entries[INVALID_ENTRY]) /* ---------- * pglz_hist_idx - @@ -272,13 +403,380 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * find 3-character matches; they very possibly will be in the wrong * hash list. This seems an acceptable tradeoff for spreading out the * hash keys more. + * + * Multiplication by the golden-ratio constant spreads similar sequences + * across the table. The high 13 bits select among the maximum 8192 lists; + * the mask adapts that result for the smaller tables used with short inputs. * ---------- */ -#define pglz_hist_idx(_s,_e, _mask) ( \ - ((((_e) - (_s)) < 4) ? (int) (_s)[0] : \ - (((_s)[0] << 6) ^ ((_s)[1] << 4) ^ \ - ((_s)[2] << 2) ^ (_s)[3])) & (_mask) \ - ) +static inline uint32 +pglz_hist_sequence(const char *s) +{ + return ((uint32) (uint8) s[0]) | + ((uint32) (uint8) s[1] << 8) | + ((uint32) (uint8) s[2] << 16) | + ((uint32) (uint8) s[3] << 24); +} + +static inline int +pglz_hist_idx(uint32 sequence, int mask) +{ + uint32 hash = sequence * 0x9E3779B1U; + + return (hash >> 19) & mask; +} + +static inline int +pglz_hist_advance(uint32 *sequence, const char *s, const char *end, int mask) +{ + if (end - s < 4) + { + *sequence = (uint8) s[0]; + return *sequence & mask; + } + + /* Drop the previous first byte and append the new fourth byte. */ + *sequence = (*sequence >> 8) | ((uint32) (uint8) s[3] << 24); + return pglz_hist_idx(*sequence, mask); +} + +static inline bool +pglz_prefix_has_nul(const char *source) +{ + const Size zero_ones = ~(Size) 0 / UCHAR_MAX; + const Size zero_highs = zero_ones << (BITS_PER_BYTE - 1); + + /* Detect zero bytes a machine word at a time without a library call. */ + for (int i = 0; i < PGLZ_PROBE_SAMPLE_SIZE; i += sizeof(Size)) + { + Size word; + + memcpy(&word, source + i, sizeof(word)); + if (((word - zero_ones) & ~word & zero_highs) != 0) + return true; + } + + return false; +} + +static inline void +pglz_bitwriter_init(PGLZ_BitWriter *writer, unsigned char *dest, + unsigned char *end) +{ + writer->ptr = dest; + writer->end = end; + writer->bits = 0; + writer->nbits = 0; + writer->failed = false; +} + +static inline void +pglz_bitwriter_put(PGLZ_BitWriter *writer, uint32 value, int nbits) +{ + uint64 mask; + + Assert(nbits > 0 && nbits <= 24); + mask = (((uint64) 1) << nbits) - 1; + writer->bits = (writer->bits << nbits) | (value & mask); + writer->nbits += nbits; + + while (writer->nbits >= 8) + { + int shift = writer->nbits - 8; + + if (writer->ptr >= writer->end) + { + writer->failed = true; + return; + } + *writer->ptr++ = (unsigned char) (writer->bits >> shift); + writer->nbits -= 8; + if (writer->nbits == 0) + writer->bits = 0; + else + writer->bits &= (((uint64) 1) << writer->nbits) - 1; + } +} + +static inline bool +pglz_bitwriter_finish(PGLZ_BitWriter *writer) +{ + if (writer->failed) + return false; + if (writer->nbits > 0) + { + if (writer->ptr >= writer->end) + return false; + *writer->ptr++ = (unsigned char) (writer->bits << (8 - writer->nbits)); + writer->bits = 0; + writer->nbits = 0; + } + return true; +} + +static inline void +pglz_bitreader_init(PGLZ_BitReader *reader, const unsigned char *source, + const unsigned char *end) +{ + reader->ptr = source; + reader->end = end; + reader->bits = 0; + reader->nbits = 0; +} + +/* Return an index padded with zero bits without consuming it. */ +static inline uint32 +pglz_bitreader_peek_padded(PGLZ_BitReader *reader, int nbits) +{ + uint64 mask; + + Assert(nbits > 0 && nbits <= 24); + if (reader->nbits < nbits && reader->end - reader->ptr >= 4) + { + uint32 word; + + memcpy(&word, reader->ptr, sizeof(word)); + reader->bits = (reader->bits << 32) | pg_ntoh32(word); + reader->ptr += sizeof(word); + reader->nbits += 32; + } + while (reader->nbits < nbits && reader->ptr < reader->end) + { + reader->bits = (reader->bits << 8) | *reader->ptr++; + reader->nbits += 8; + } + mask = (((uint64) 1) << nbits) - 1; + if (reader->nbits >= nbits) + return (reader->bits >> (reader->nbits - nbits)) & mask; + if (reader->nbits == 0) + return 0; + return (reader->bits & ((((uint64) 1) << reader->nbits) - 1)) << + (nbits - reader->nbits); +} + +static inline bool +pglz_bitreader_drop(PGLZ_BitReader *reader, int nbits) +{ + if (reader->nbits < nbits) + return false; + reader->nbits -= nbits; + return true; +} + +static inline bool +pglz_huff_node_less(const PGLZ_HuffNode *nodes, int left, int right) +{ + if (nodes[left].frequency != nodes[right].frequency) + return nodes[left].frequency < nodes[right].frequency; + return nodes[left].min_symbol < nodes[right].min_symbol; +} + +static void +pglz_huff_heap_push(int *heap, int *heap_size, const PGLZ_HuffNode *nodes, + int node) +{ + int child = ++(*heap_size); + + while (child > 1) + { + int parent = child / 2; + + if (!pglz_huff_node_less(nodes, node, heap[parent])) + break; + heap[child] = heap[parent]; + child = parent; + } + heap[child] = node; +} + +static int +pglz_huff_heap_pop(int *heap, int *heap_size, const PGLZ_HuffNode *nodes) +{ + int result = heap[1]; + int last = heap[(*heap_size)--]; + int parent = 1; + + while (parent * 2 <= *heap_size) + { + int child = parent * 2; + + if (child < *heap_size && + pglz_huff_node_less(nodes, heap[child + 1], heap[child])) + child++; + if (!pglz_huff_node_less(nodes, heap[child], last)) + break; + heap[parent] = heap[child]; + parent = child; + } + if (*heap_size > 0) + heap[parent] = last; + return result; +} + +/* Validate lengths and assign canonical Huffman codes. */ +static bool +pglz_huffman_codes(const uint8 *lengths, uint16 *codes, int *nsymbolsp) +{ + int bit_count[PGLZ_V2_HUFF_BITS + 1] = {0}; + int next_code[PGLZ_V2_HUFF_BITS + 1] = {0}; + int code = 0; + int nsymbols = 0; + + for (int sym = 0; sym < 256; sym++) + { + if (lengths[sym] > PGLZ_V2_HUFF_BITS) + return false; + if (lengths[sym] > 0) + { + bit_count[lengths[sym]]++; + nsymbols++; + } + } + if (nsymbols == 0) + return false; + + for (int bits = 1; bits <= PGLZ_V2_HUFF_BITS; bits++) + { + code = (code + bit_count[bits - 1]) << 1; + if (code + bit_count[bits] > (1 << bits)) + return false; + next_code[bits] = code; + } + if (nsymbols > 1 && + code + bit_count[PGLZ_V2_HUFF_BITS] != + (1 << PGLZ_V2_HUFF_BITS)) + return false; + + for (int sym = 0; sym < 256; sym++) + { + int length = lengths[sym]; + + if (length > 0) + codes[sym] = next_code[length]++; + } + *nsymbolsp = nsymbols; + return true; +} + +static bool +pglz_build_huffman(const uint32 *frequencies, uint8 *lengths, uint16 *codes, + int *nsymbols) +{ + PGLZ_HuffNode nodes[511]; + int heap[512]; + int bit_count[256] = {0}; + int sorted_symbols[256]; + int16 leaf[256]; + int heap_size = 0; + int nnode = 0; + int symbol_count = 0; + int leaf_count; + int overflow = 0; + + memset(lengths, 0, 256 * sizeof(uint8)); + for (int sym = 0; sym < 256; sym++) + { + leaf[sym] = -1; + if (frequencies[sym] == 0) + continue; + nodes[nnode].frequency = frequencies[sym]; + nodes[nnode].parent = -1; + nodes[nnode].min_symbol = sym; + leaf[sym] = nnode; + sorted_symbols[symbol_count++] = sym; + pglz_huff_heap_push(heap, &heap_size, nodes, nnode++); + } + + if (heap_size == 0) + return false; + if (heap_size == 1) + { + lengths[nodes[heap[1]].min_symbol] = 1; + return pglz_huffman_codes(lengths, codes, nsymbols); + } + + while (heap_size > 1) + { + int left = pglz_huff_heap_pop(heap, &heap_size, nodes); + int right = pglz_huff_heap_pop(heap, &heap_size, nodes); + + nodes[left].parent = nnode; + nodes[right].parent = nnode; + nodes[nnode].frequency = nodes[left].frequency + nodes[right].frequency; + nodes[nnode].parent = -1; + nodes[nnode].min_symbol = Min(nodes[left].min_symbol, + nodes[right].min_symbol); + pglz_huff_heap_push(heap, &heap_size, nodes, nnode++); + } + + for (int i = 1; i < symbol_count; i++) + { + int sym = sorted_symbols[i]; + int j = i; + + while (j > 0) + { + int previous = sorted_symbols[j - 1]; + + if (frequencies[previous] < frequencies[sym] || + (frequencies[previous] == frequencies[sym] && + previous < sym)) + break; + sorted_symbols[j] = previous; + j--; + } + sorted_symbols[j] = sym; + } + + for (int i = 0; i < symbol_count; i++) + { + int depth = 0; + int node = leaf[sorted_symbols[i]]; + + while (nodes[node].parent >= 0) + { + depth++; + node = nodes[node].parent; + } + if (depth > PGLZ_V2_HUFF_BITS) + { + depth = PGLZ_V2_HUFF_BITS; + overflow++; + } + bit_count[depth]++; + } + + /* Restore a complete tree after clamping overlong codes. */ + while (overflow > 0) + { + int bits = PGLZ_V2_HUFF_BITS - 1; + + while (bits > 0 && bit_count[bits] == 0) + bits--; + if (bits == 0 || bit_count[PGLZ_V2_HUFF_BITS] == 0) + return false; + bit_count[bits]--; + bit_count[bits + 1] += 2; + bit_count[PGLZ_V2_HUFF_BITS]--; + overflow -= 2; + } + + leaf_count = symbol_count; + symbol_count = 0; + for (int bits = PGLZ_V2_HUFF_BITS; bits > 0; bits--) + { + for (int i = 0; i < bit_count[bits]; i++) + { + if (symbol_count >= leaf_count) + return false; + lengths[sorted_symbols[symbol_count++]] = bits; + } + } + if (symbol_count != leaf_count) + return false; + + return pglz_huffman_codes(lengths, codes, nsymbols); +} /* ---------- @@ -286,45 +784,22 @@ static PGLZ_HistEntry hist_entries[PGLZ_HISTORY_SIZE + 1]; * * Adds a new entry to the history table. * - * If _recycle is true, then we are recycling a previously used entry, - * and must first delink it from its old hashcode's linked list. - * - * NOTE: beware of multiple evaluations of macro's arguments, and note that - * _hn and _recycle are modified in the macro. * ---------- */ -#define pglz_hist_add(_hs,_he,_hn,_recycle,_s,_e, _mask) \ -do { \ - int __hindex = pglz_hist_idx((_s),(_e), (_mask)); \ - int16 *__myhsp = &(_hs)[__hindex]; \ - PGLZ_HistEntry *__myhe = &(_he)[_hn]; \ - if (_recycle) { \ - if (__myhe->prev == NULL) \ - (_hs)[__myhe->hindex] = __myhe->next - (_he); \ - else \ - __myhe->prev->next = __myhe->next; \ - if (__myhe->next != NULL) \ - __myhe->next->prev = __myhe->prev; \ - } \ - __myhe->next = &(_he)[*__myhsp]; \ - __myhe->prev = NULL; \ - __myhe->hindex = __hindex; \ - __myhe->pos = (_s); \ - /* If there was an existing entry in this hash slot, link */ \ - /* this new entry to it. However, the 0th entry in the */ \ - /* entries table is unused, so we can freely scribble on it. */ \ - /* So don't bother checking if the slot was used - we'll */ \ - /* scribble on the unused entry if it was not, but that's */ \ - /* harmless. Avoiding the branch in this critical path */ \ - /* speeds this up a little bit. */ \ - /* if (*__myhsp != INVALID_ENTRY) */ \ - (_he)[(*__myhsp)].prev = __myhe; \ - *__myhsp = _hn; \ - if (++(_hn) >= PGLZ_HISTORY_SIZE + 1) { \ - (_hn) = 1; \ - (_recycle) = true; \ - } \ -} while (0) +static inline void +pglz_hist_add(uint16 *hstart, PGLZ_HistEntry *hentries, int *hist_next, + int hindex) +{ + uint16 *head = &hstart[hindex]; + PGLZ_HistEntry *entry = &hentries[*hist_next]; + + entry->next = *head; + entry->hindex = hindex; + *head = *hist_next; + + if (++(*hist_next) >= PGLZ_HISTORY_SIZE + 1) + *hist_next = 1; +} /* ---------- @@ -387,6 +862,723 @@ do { \ } while (0) +/* Return the byte position of the first difference in two native words. */ +static inline int32 +pglz_word_mismatch(Size diff) +{ +#ifdef WORDS_BIGENDIAN + return sizeof(Size) - 1 - + (pg_leftmost_one_pos_size_t(diff) / BITS_PER_BYTE); +#else +#if SIZEOF_SIZE_T == 4 + return pg_rightmost_one_pos32((uint32) diff) / BITS_PER_BYTE; +#else + return pg_rightmost_one_pos64((uint64) diff) / BITS_PER_BYTE; +#endif +#endif +} + +/* + * Determine the exact length of a match, stopping at matchend. Compare a + * vector at a time where PostgreSQL has a SIMD implementation, then compare + * native words and finally individual bytes. The word loads use memcpy() so + * they do not require aligned input pointers. + */ +static inline int32 +pglz_match_len_fallback(const char *ip, const char *hp, const char *matchend) +{ + const char *start = ip; + +#ifndef USE_NO_SIMD + while (matchend - ip >= (int) sizeof(Vector8)) + { + Vector8 iv; + Vector8 hv; + uint32 mask; + uint32 allmatch; + + vector8_load(&iv, (const uint8 *) ip); + vector8_load(&hv, (const uint8 *) hp); + mask = vector8_highbit_mask(vector8_eq(iv, hv)); + allmatch = UINT32_MAX >> (32 - sizeof(Vector8)); + + if (mask != allmatch) + return (ip - start) + + pg_rightmost_one_pos32((~mask) & allmatch); + + ip += sizeof(Vector8); + hp += sizeof(Vector8); + } +#endif + + while (matchend - ip >= (int) sizeof(Size)) + { + Size ival; + Size hval; + Size diff; + + memcpy(&ival, ip, sizeof(Size)); + memcpy(&hval, hp, sizeof(Size)); + diff = ival ^ hval; + + if (diff != 0) + return (ip - start) + pglz_word_mismatch(diff); + + ip += sizeof(Size); + hp += sizeof(Size); + } + + while (ip < matchend && *ip == *hp) + { + ip++; + hp++; + } + + return ip - start; +} + +#ifdef USE_AVX2_WITH_RUNTIME_CHECK +/* AVX2-specialized match extension, selected after a runtime CPU check. */ +pg_attribute_target("avx2") +static int32 +pglz_match_len_avx2(const char *ip, const char *hp, const char *matchend) +{ + const char *start = ip; + + while (matchend - ip >= (int) sizeof(__m256i)) + { + __m256i iv; + __m256i hv; + uint32 mask; + + iv = _mm256_loadu_si256((const __m256i *) ip); + hv = _mm256_loadu_si256((const __m256i *) hp); + mask = (uint32) _mm256_movemask_epi8(_mm256_cmpeq_epi8(iv, hv)); + + if (mask != UINT32_MAX) + return (ip - start) + pg_rightmost_one_pos32(~mask); + + ip += sizeof(__m256i); + hp += sizeof(__m256i); + } + + return (ip - start) + pglz_match_len_fallback(ip, hp, matchend); +} +#endif /* USE_AVX2_WITH_RUNTIME_CHECK */ + +static inline int32 +pglz_match_len(const char *ip, const char *hp, const char *end, int32 maxlen, + bool use_avx2) +{ + const char *start = ip; + const char *matchend; + + if (end - ip > maxlen) + matchend = ip + maxlen; + else + matchend = end; + + /* Most hash collisions differ within the first native word. */ + if (matchend - ip >= (int) sizeof(Size)) + { + Size ival; + Size hval; + Size diff; + + memcpy(&ival, ip, sizeof(Size)); + memcpy(&hval, hp, sizeof(Size)); + diff = ival ^ hval; + + if (diff != 0) + return pglz_word_mismatch(diff); + + ip += sizeof(Size); + hp += sizeof(Size); + } + +#ifdef USE_AVX2_WITH_RUNTIME_CHECK + if (use_avx2 && matchend - ip >= (int) sizeof(__m256i)) + return (ip - start) + + pglz_match_len_avx2(ip, hp, matchend); +#else + (void) use_avx2; +#endif + + return (ip - start) + + pglz_match_len_fallback(ip, hp, matchend); +} + +static inline bool +pglz_token_begin(PGLZ_TokenWriter *writer) +{ + if (writer->items_left == 0) + { + if (writer->ptr >= writer->end) + return false; + writer->control = writer->ptr++; + *writer->control = 0; + writer->mask = 1; + writer->items_left = 8; + } + return true; +} + +static inline void +pglz_token_advance(PGLZ_TokenWriter *writer) +{ + writer->mask <<= 1; + writer->items_left--; +} + +static inline bool +pglz_token_literal(PGLZ_TokenWriter *writer, unsigned char literal) +{ + if (!pglz_token_begin(writer) || writer->ptr >= writer->end) + return false; + *writer->ptr++ = literal; + pglz_token_advance(writer); + return true; +} + +static inline bool +pglz_token_match(PGLZ_TokenWriter *writer, int32 length, int32 offset) +{ + if (!pglz_token_begin(writer) || writer->end - writer->ptr < 4) + return false; + *writer->control |= writer->mask; + writer->ptr[0] = offset & 0xff; + writer->ptr[1] = (offset >> 8) & 0xff; + writer->ptr[2] = length & 0xff; + writer->ptr[3] = (length >> 8) & 0xff; + writer->ptr += 4; + pglz_token_advance(writer); + return true; +} + +static inline void +pglz_token_reader_init(PGLZ_TokenReader *reader, + const unsigned char *tokens, + const unsigned char *token_end) +{ + reader->ptr = tokens; + reader->end = token_end; + reader->control = 0; + reader->items_left = 0; +} + +static inline bool +pglz_token_read(PGLZ_TokenReader *reader, bool *is_match, + unsigned char *literal, int32 *length, int32 *offset) +{ + if (reader->items_left == 0) + { + if (reader->ptr >= reader->end) + return false; + reader->control = *reader->ptr++; + reader->items_left = 8; + } + + *is_match = (reader->control & 1) != 0; + if (*is_match) + { + if (reader->end - reader->ptr < 4) + return false; + *offset = reader->ptr[0] | (reader->ptr[1] << 8); + *length = reader->ptr[2] | (reader->ptr[3] << 8); + reader->ptr += 4; + } + else + { + if (reader->ptr >= reader->end) + return false; + *literal = *reader->ptr++; + } + reader->control >>= 1; + reader->items_left--; + return true; +} + +static inline int +pglz_v2_hash(const char *source) +{ + uint32 sequence = pglz_hist_sequence(source); + + return (sequence * 0x9E3779B1U) >> (32 - PGLZ_V2_HASH_BITS); +} + +static inline void +pglz_v2_hash_position(uint32 *hash_table, const char *source, int32 slen, + int32 position) +{ + if (position >= 0 && position <= slen - 4) + hash_table[pglz_v2_hash(source + position)] = position + 1; +} + +static pg_always_inline int32 +pglz_modern_find_match(uint32 *hash_table, const char *source, + const char *source_end, int32 position, + int32 *match_offset, bool use_avx2) +{ + int hindex = pglz_v2_hash(source + position); + uint32 entry = hash_table[hindex]; + + hash_table[hindex] = position + 1; + if (entry != 0) + { + int32 candidate = entry - 1; + + *match_offset = position - candidate; + if (*match_offset > 0 && *match_offset <= PGLZ_V2_WINDOW_SIZE) + return pglz_match_len(source + position, source + candidate, + source_end, PGLZ_V2_MAX_MATCH, use_avx2); + } + + *match_offset = 0; + return 0; +} + +static inline void +pglz_modern_hash_match(uint32 *hash_table, const char *source, int32 slen, + int32 match_start, int32 match_end) +{ + /* Keep a few positions around both match boundaries searchable. */ + pglz_v2_hash_position(hash_table, source, slen, match_start + 1); + pglz_v2_hash_position(hash_table, source, slen, match_start + 2); + pglz_v2_hash_position(hash_table, source, slen, match_end - 2); + pglz_v2_hash_position(hash_table, source, slen, match_end - 1); +} + +/* Build a compact token stream for the version 2 fallback encoder. */ +static int32 +pglz_v2_tokenize(const char *source, int32 slen, unsigned char *tokens, + unsigned char *token_end, uint32 *hash_table, + int32 first_success_by, bool use_avx2) +{ + PGLZ_TokenWriter writer; + const char *source_end = source + slen; + int32 position = 0; + bool found_match = false; + + writer.ptr = tokens; + writer.end = token_end; + writer.control = NULL; + writer.mask = 0; + writer.items_left = 0; + + while (position <= slen - 4) + { + int32 match_length; + int32 match_offset = 0; + + match_length = pglz_modern_find_match(hash_table, source, source_end, + position, &match_offset, use_avx2); + + if (match_length >= PGLZ_V2_MIN_MATCH) + { + int32 match_start = position; + + if (!pglz_token_match(&writer, match_length, match_offset)) + return -1; + position += match_length; + found_match = true; + + pglz_modern_hash_match(hash_table, source, slen, match_start, + position); + } + else + { + unsigned char literal = (unsigned char) source[position++]; + + if (!pglz_token_literal(&writer, literal)) + return -1; + } + + if (!found_match && position >= first_success_by) + return -1; + } + + while (position < slen) + { + unsigned char literal = (unsigned char) source[position++]; + + if (!pglz_token_literal(&writer, literal)) + return -1; + } + + return writer.ptr - tokens; +} + +static inline bool +pglz_v3_put_varint(unsigned char **ptr, unsigned char *end, uint32 value) +{ + do + { + unsigned char byte = value & 0x7f; + + value >>= 7; + if (value != 0) + byte |= 0x80; + if (*ptr >= end) + return false; + *(*ptr)++ = byte; + } while (value != 0); + return true; +} + +static bool +pglz_v3_emit_sequence(unsigned char **dest, unsigned char *dest_end, + const char *literals, int32 literal_length, + int32 match_length, int32 match_offset) +{ + unsigned char *bp = *dest; + int32 match_code = match_length - PGLZ_V2_MIN_MATCH; + unsigned char header; + + if (literal_length < 0 || match_length < 0 || + (match_length != 0 && + (match_code < 0 || match_length > PGLZ_V2_MAX_MATCH || + match_offset <= 0 || match_offset > PGLZ_V2_WINDOW_SIZE))) + return false; + + header = Min(literal_length, PGLZ_V3_LITERAL_MASK) << + PGLZ_V3_LITERAL_SHIFT; + if (match_length != 0) + header |= Min(match_code, PGLZ_V3_MATCH_MASK); + if (bp >= dest_end) + return false; + *bp++ = header; + if (literal_length >= PGLZ_V3_LITERAL_MASK && + !pglz_v3_put_varint(&bp, dest_end, + literal_length - PGLZ_V3_LITERAL_MASK)) + return false; + if (dest_end - bp < literal_length) + return false; + memcpy(bp, literals, literal_length); + bp += literal_length; + + if (match_length != 0) + { + if (dest_end - bp < 2) + return false; + bp[0] = (match_offset - 1) & 0xff; + bp[1] = (match_offset - 1) >> 8; + bp += 2; + if (match_code >= PGLZ_V3_MATCH_MASK && + !pglz_v3_put_varint(&bp, dest_end, + match_code - PGLZ_V3_MATCH_MASK)) + return false; + } + + *dest = bp; + return true; +} + +/* Parse and encode version 3 directly, without an intermediate token stream. */ +static int32 +pglz_v3_compress(const char *source, int32 slen, char *dest, + uint32 *hash_table, int32 result_max, + int32 first_success_by, bool use_avx2) +{ + const char *source_end = source + slen; + const char *literal_start = source; + unsigned char *bp = (unsigned char *) dest; + unsigned char *dest_end = bp + result_max; + int32 position = 0; + bool found_match = false; + + if (dest_end - bp < (int) sizeof(pglz_v3_magic) + 1) + return -1; + memcpy(bp, pglz_v3_magic, sizeof(pglz_v3_magic)); + bp += sizeof(pglz_v3_magic); + + while (position <= slen - 4) + { + int32 match_length; + int32 match_offset = 0; + + match_length = pglz_modern_find_match(hash_table, source, source_end, + position, &match_offset, use_avx2); + + if (match_length >= PGLZ_V2_MIN_MATCH) + { + int32 match_start = position; + int32 literal_length = source + position - literal_start; + + if (!pglz_v3_emit_sequence(&bp, dest_end, literal_start, + literal_length, match_length, + match_offset)) + return -1; + position += match_length; + literal_start = source + position; + found_match = true; + + pglz_modern_hash_match(hash_table, source, slen, match_start, + position); + } + else + position++; + + if (!found_match && position >= first_success_by) + return PGLZ_V3_NO_MATCH; + } + + if (literal_start < source_end && + !pglz_v3_emit_sequence(&bp, dest_end, literal_start, + source_end - literal_start, 0, 0)) + return -1; + if (bp - (unsigned char *) dest >= result_max) + return -1; + return bp - (unsigned char *) dest; +} + +static bool +pglz_token_frequencies(const unsigned char *tokens, + const unsigned char *token_end, + uint32 *frequencies) +{ + PGLZ_TokenReader reader; + unsigned char literal = 0; + int32 length = 0; + int32 offset = 0; + bool is_match; + + pglz_token_reader_init(&reader, tokens, token_end); + while (pglz_token_read(&reader, &is_match, &literal, &length, &offset)) + { + if (!is_match) + frequencies[literal]++; + } + return reader.ptr == reader.end; +} + +static inline void +pglz_v2_put_match(PGLZ_BitWriter *writer, int32 length, int32 offset) +{ + int32 extra; + + pglz_bitwriter_put(writer, 1, 1); + if (offset <= 256) + { + pglz_bitwriter_put(writer, 0, 1); + pglz_bitwriter_put(writer, offset - 1, 8); + } + else + { + pglz_bitwriter_put(writer, 1, 1); + pglz_bitwriter_put(writer, offset - 1, 16); + } + + if (length - PGLZ_V2_MIN_MATCH < 15) + { + pglz_bitwriter_put(writer, length - PGLZ_V2_MIN_MATCH, 4); + return; + } + + pglz_bitwriter_put(writer, 15, 4); + extra = length - PGLZ_V2_MIN_MATCH - 15; + do + { + uint32 byte = extra & 0x7f; + + extra >>= 7; + if (extra != 0) + byte |= 0x80; + pglz_bitwriter_put(writer, byte, 8); + } while (extra != 0); +} + +static int32 +pglz_v2_encode(const unsigned char *tokens, int32 token_size, int32 rawsize, + char *dest, int32 result_max, const uint8 *lengths, + const uint16 *codes, int nsymbols) +{ + unsigned char *bp = (unsigned char *) dest; + unsigned char *dest_end = bp + result_max; + unsigned char *bitmap; + unsigned char *length_data; + const unsigned char *tp = tokens; + const unsigned char *token_end = tokens + token_size; + PGLZ_BitWriter writer; + int length_index = 0; + int32 raw_position = 0; + unsigned char control = 0; + int items_left = 0; + + if (dest_end - bp < PGLZ_V2_MAX_HEADER) + return -1; + memcpy(bp, pglz_v2_magic, sizeof(pglz_v2_magic)); + bp += sizeof(pglz_v2_magic); + bitmap = bp; + memset(bitmap, 0, PGLZ_V2_BITMAP_SIZE); + bp += PGLZ_V2_BITMAP_SIZE; + length_data = bp; + memset(length_data, 0, (nsymbols + 1) / 2); + + for (int sym = 0; sym < 256; sym++) + { + if (lengths[sym] == 0) + continue; + bitmap[sym >> 3] |= 1U << (sym & 7); + if ((length_index & 1) == 0) + length_data[length_index >> 1] = lengths[sym]; + else + length_data[length_index >> 1] |= lengths[sym] << 4; + length_index++; + } + bp += (nsymbols + 1) / 2; + pglz_bitwriter_init(&writer, bp, dest_end); + + while (raw_position < rawsize) + { + if (items_left == 0) + { + if (tp >= token_end) + return -1; + control = *tp++; + items_left = 8; + } + + if (control & 1) + { + int32 offset; + int32 length; + + if (token_end - tp < 4) + return -1; + offset = tp[0] | (tp[1] << 8); + length = tp[2] | (tp[3] << 8); + tp += 4; + if (offset <= 0 || + offset > Min(raw_position, PGLZ_V2_WINDOW_SIZE) || + length < PGLZ_V2_MIN_MATCH || + length > PGLZ_V2_MAX_MATCH || + raw_position + length > rawsize) + return -1; + pglz_v2_put_match(&writer, length, offset); + raw_position += length; + } + else + { + unsigned char literal; + + if (tp >= token_end) + return -1; + literal = *tp++; + pglz_bitwriter_put(&writer, 0, 1); + pglz_bitwriter_put(&writer, codes[literal], lengths[literal]); + raw_position++; + } + if (writer.failed) + return -1; + control >>= 1; + items_left--; + } + + if (tp != token_end || !pglz_bitwriter_finish(&writer)) + return -1; + return writer.ptr - (unsigned char *) dest; +} + +static int32 +pglz_compress_modern(const char *source, int32 slen, char *dest, + const PGLZ_Strategy *strategy) +{ + uint32 frequencies[256] = {0}; + uint8 lengths[256]; + uint16 codes[256]; + unsigned char *tokens; + uint32 *hash_table; + int64 token_max64; + int32 token_size; + int32 result_size; + int32 result_max; + int32 need_rate; + int nsymbols; + bool use_avx2 = false; + + if (strategy == NULL) + strategy = PGLZ_strategy_default; + if (strategy->match_size_good <= 0 || + slen < Max(strategy->min_input_size, PGLZ_V2_MIN_INPUT) || + slen > strategy->max_input_size) + return -1; + + need_rate = strategy->min_comp_rate; + if (need_rate < 0) + need_rate = 0; + else if (need_rate > 99) + need_rate = 99; + if (slen > (INT_MAX / 100)) + result_max = (slen / 100) * (100 - need_rate); + else + result_max = (slen * (100 - need_rate)) / 100; + if (result_max <= PGLZ_V3_MAX_HEADER) + return -1; + + hash_table = (uint32 *) PGLZ_ALLOC(PGLZ_V2_HASH_SIZE * sizeof(uint32)); + if (hash_table == NULL) + return -1; + memset(hash_table, 0, PGLZ_V2_HASH_SIZE * sizeof(uint32)); + +#ifdef USE_AVX2_WITH_RUNTIME_CHECK + use_avx2 = x86_feature_available(PG_AVX2); +#endif + result_size = pglz_v3_compress(source, slen, dest, hash_table, result_max, + strategy->first_success_by, use_avx2); + if (result_size >= 0) + { + PGLZ_FREE(hash_table); + return result_size; + } + if (result_size == PGLZ_V3_NO_MATCH || + result_max <= PGLZ_V2_MAX_HEADER) + { + PGLZ_FREE(hash_table); + return -1; + } + + /* + * Keep the common version 3 path free of an intermediate token buffer. + * Allocate and rebuild the parse only when its byte-aligned stream could + * not meet the requested compression rate and version 2 might do so. One + * control bit plus one byte per literal is the token worst case. + */ + token_max64 = ((int64) slen * 9 + 7) / 8 + 1; + if (token_max64 > (int64) MaxAllocSize) + { + PGLZ_FREE(hash_table); + return -1; + } + tokens = (unsigned char *) PGLZ_ALLOC((Size) token_max64); + if (tokens == NULL) + { + PGLZ_FREE(hash_table); + return -1; + } + memset(hash_table, 0, PGLZ_V2_HASH_SIZE * sizeof(uint32)); + token_size = pglz_v2_tokenize(source, slen, tokens, + tokens + token_max64, hash_table, + strategy->first_success_by, + use_avx2); + + if (token_size < 0 || + !pglz_token_frequencies(tokens, tokens + token_size, frequencies) || + !pglz_build_huffman(frequencies, lengths, codes, &nsymbols)) + { + PGLZ_FREE(tokens); + PGLZ_FREE(hash_table); + return -1; + } + result_size = pglz_v2_encode(tokens, token_size, slen, dest, result_max, + lengths, codes, nsymbols); + PGLZ_FREE(tokens); + PGLZ_FREE(hash_table); + if (result_size < 0 || result_size >= result_max) + return -1; + return result_size; +} + + /* ---------- * pglz_find_match - * @@ -395,67 +1587,92 @@ do { \ * in the input buffer. * ---------- */ -static inline int -pglz_find_match(int16 *hstart, const char *input, const char *end, - int *lenp, int *offp, int good_match, int good_drop, int mask) +static pg_always_inline int +pglz_find_match_internal(uint16 *hstart, const char *input, const char *end, + int hist_next, int hindex, int probe_limit, + int *lenp, int *offp, int good_match, int good_drop, + bool use_avx2) { - PGLZ_HistEntry *hent; - int16 hentno; + uint16 hentno; int32 len = 0; int32 off = 0; + int32 previousoff = 0; + int probes = 0; /* * Traverse the linked history list until a good enough match is found. */ - hentno = hstart[pglz_hist_idx(input, end, mask)]; - hent = &hist_entries[hentno]; - while (hent != INVALID_ENTRY_PTR) + hentno = hstart[hindex]; + while (hentno != INVALID_ENTRY) { + PGLZ_HistEntry *hent = &hist_entries[hentno]; const char *ip = input; - const char *hp = hent->pos; + const char *hp; int32 thisoff; int32 thislen; /* - * Stop if the offset does not fit into our tag anymore. + * Since one ring entry is added per input byte, the modular distance + * between hist_next and hentno is also the backward offset. A link + * to an overwritten ring entry either has a different hash key or + * fails to increase the offset as we walk from newer to older + * entries. */ - thisoff = ip - hp; + thisoff = hist_next - hentno; + if (thisoff <= 0) + thisoff += PGLZ_HISTORY_SIZE; + if (hent->hindex != hindex || thisoff <= previousoff) + break; if (thisoff >= 0x0fff) break; + hp = input - thisoff; + + /* + * A candidate can only improve on the best match if its next byte + * also matches. Check that byte before comparing the whole prefix; + * this rejects most hash collisions and many shorter matches cheaply. + */ + if (len < PGLZ_MAX_MATCH && len < end - input && + hp[len] != input[len]) + goto next_candidate; /* * Determine length of match. A better match must be larger than the * best so far. And if we already have a match of 16 or more bytes, * it's worth the call overhead to use memcmp() to check if this match - * is equal for the same size. After that we must fallback to - * character by character comparison to know the exact position where - * the diff occurred. + * is equal for the same size. After that, extend the match using the + * best implementation available on this CPU. */ thislen = 0; if (len >= 16) { + if (probe_limit == PGLZ_FAST_MATCH_PROBES) + { + Size input_tail; + Size history_tail; + + /* + * Reject a differing tail word before comparing the whole + * prefix. + */ + memcpy(&input_tail, ip + len - sizeof(Size), sizeof(Size)); + memcpy(&history_tail, hp + len - sizeof(Size), sizeof(Size)); + if (input_tail != history_tail) + goto next_candidate; + } if (memcmp(ip, hp, len) == 0) { thislen = len; ip += len; hp += len; - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } + thislen += pglz_match_len(ip, hp, end, + PGLZ_MAX_MATCH - thislen, + use_avx2); } } else - { - while (ip < end && *ip == *hp && thislen < PGLZ_MAX_MATCH) - { - thislen++; - ip++; - hp++; - } - } + thislen = pglz_match_len(ip, hp, end, PGLZ_MAX_MATCH, + use_avx2); /* * Remember this match as the best (if it is) @@ -466,16 +1683,28 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, off = thisoff; } +next_candidate: + /* * Advance to the next history entry */ - hent = hent->next; + previousoff = thisoff; + hentno = hent->next; + + /* + * Long history chains can dominate compression time. Newer entries + * usually provide the useful matches. If the good-match heuristic + * did not stop first, stop when this input's candidate budget is + * exhausted. + */ + if (++probes >= probe_limit) + break; /* * Be happy with lesser good matches the more entries we visited. But * no point in doing calculation if we're at end of list. */ - if (hent != INVALID_ENTRY_PTR) + if (hentno != INVALID_ENTRY) { if (len >= good_match) break; @@ -499,20 +1728,21 @@ pglz_find_match(int16 *hstart, const char *input, const char *end, /* ---------- - * pglz_compress - + * pglz_compress_internal - * * Compresses source into dest using strategy. Returns the number of * bytes written in buffer dest, or -1 if compression fails. + * + * Always inline so the specialized callers have a constant probe limit. * ---------- */ -int32 -pglz_compress(const char *source, int32 slen, char *dest, - const PGLZ_Strategy *strategy) +static pg_always_inline int32 +pglz_compress_internal(const char *source, int32 slen, char *dest, + const PGLZ_Strategy *strategy, int probe_limit) { unsigned char *bp = (unsigned char *) dest; unsigned char *bstart = bp; int hist_next = 1; - bool hist_recycle = false; const char *dp = source; const char *dend = source + slen; unsigned char ctrl_dummy = 0; @@ -520,6 +1750,7 @@ pglz_compress(const char *source, int32 slen, char *dest, unsigned char ctrlb = 0; unsigned char ctrl = 0; bool found_match = false; + bool use_avx2 = false; int32 match_len; int32 match_off; int32 good_match; @@ -527,7 +1758,9 @@ pglz_compress(const char *source, int32 slen, char *dest, int32 result_size; int32 result_max; int32 need_rate; + uint32 hist_sequence = 0; int hashsz; + int hindex = 0; int mask; /* @@ -566,6 +1799,11 @@ pglz_compress(const char *source, int32 slen, char *dest, else if (need_rate > 99) need_rate = 99; +#ifdef USE_AVX2_WITH_RUNTIME_CHECK + /* Do runtime feature detection once per input, not once per match. */ + use_avx2 = x86_feature_available(PG_AVX2); +#endif + /* * Compute the maximum result size allowed by the strategy, namely the * input size minus the minimum wanted compression rate. This had better @@ -601,13 +1839,30 @@ pglz_compress(const char *source, int32 slen, char *dest, * Initialize the history lists to empty. We do not need to zero the * hist_entries[] array; its entries are initialized as they are used. */ - memset(hist_start, 0, hashsz * sizeof(int16)); + memset(hist_start, 0, hashsz * sizeof(uint16)); + + /* Initialize the sequence used by the rolling history hash. */ + if (dp < dend) + { + if (dend - dp >= 4) + { + hist_sequence = pglz_hist_sequence(dp); + hindex = pglz_hist_idx(hist_sequence, mask); + } + else + { + hist_sequence = (uint8) dp[0]; + hindex = hist_sequence & mask; + } + } /* * Compress the source directly into the output buffer. */ while (dp < dend) { + bool match_found; + /* * If we already exceeded the maximum result size, fail. * @@ -630,8 +1885,12 @@ pglz_compress(const char *source, int32 slen, char *dest, /* * Try to find a match in the history */ - if (pglz_find_match(hist_start, dp, dend, &match_len, - &match_off, good_match, good_drop, mask)) + match_found = pglz_find_match_internal(hist_start, dp, dend, hist_next, + hindex, probe_limit, &match_len, + &match_off, good_match, good_drop, + use_avx2); + + if (match_found) { /* * Create the tag and add history entries for all matched @@ -640,11 +1899,10 @@ pglz_compress(const char *source, int32 slen, char *dest, pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off); while (match_len--) { - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + pglz_hist_add(hist_start, hist_entries, &hist_next, hindex); + dp++; + if (dp < dend) + hindex = pglz_hist_advance(&hist_sequence, dp, dend, mask); } found_match = true; } @@ -654,11 +1912,10 @@ pglz_compress(const char *source, int32 slen, char *dest, * No match found. Copy one literal byte. */ pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp); - pglz_hist_add(hist_start, hist_entries, - hist_next, hist_recycle, - dp, dend, mask); - dp++; /* Do not do this ++ in the line above! */ - /* The macro would do it four times - Jan. */ + pglz_hist_add(hist_start, hist_entries, &hist_next, hindex); + dp++; + if (dp < dend) + hindex = pglz_hist_advance(&hist_sequence, dp, dend, mask); } } @@ -676,6 +1933,595 @@ pglz_compress(const char *source, int32 slen, char *dest, } +static pg_noinline int32 +pglz_compress_full(const char *source, int32 slen, char *dest, + const PGLZ_Strategy *strategy) +{ + return pglz_compress_internal(source, slen, dest, strategy, + PGLZ_FULL_MATCH_PROBES); +} + +int32 +pglz_compress(const char *source, int32 slen, char *dest, + const PGLZ_Strategy *strategy) +{ + int32 result; + + /* + * Long inputs can benefit from the larger-window parser. Prefer the + * byte-aligned version 3 stream, then use version 2 when Huffman-coded + * literals are needed to satisfy the requested compression rate. + */ + if (slen >= PGLZ_V2_MIN_INPUT) + { + result = pglz_compress_modern(source, slen, dest, strategy); + if (result >= 0) + return result; + if (!pglz_prefix_has_nul(source)) + return pglz_compress_internal(source, slen, dest, strategy, + PGLZ_FAST_MATCH_PROBES); + } + return pglz_compress_full(source, slen, dest, strategy); +} + +static bool +pglz_huffman_decode_tables(const uint8 *lengths, uint32 *root_table, + uint16 *subtables) +{ + uint16 codes[256]; + int nsymbols; + int nsubtables = 0; + + if (!pglz_huffman_codes(lengths, codes, &nsymbols)) + return false; + memset(root_table, 0, + (1 << PGLZ_V2_DECODE_BITS) * sizeof(uint32)); + + for (int sym = 0; sym < 256; sym++) + { + int length = lengths[sym]; + int first; + int count; + uint16 entry; + + if (length == 0) + continue; + entry = (length << 8) | sym; + if (length <= PGLZ_V2_DECODE_BITS) + { + first = codes[sym] << (PGLZ_V2_DECODE_BITS - length); + count = 1 << (PGLZ_V2_DECODE_BITS - length); + for (int i = 0; i < count; i++) + { + if (root_table[first + i] != 0) + return false; + root_table[first + i] = entry; + } + } + else + { + int prefix = codes[sym] >> + (length - PGLZ_V2_DECODE_BITS); + int subtable; + int suffix_bits = length - PGLZ_V2_DECODE_BITS; + + if (root_table[prefix] == 0) + { + if (nsubtables >= PGLZ_V2_MAX_SUBTABLES) + return false; + subtable = nsubtables++; + memset(subtables + subtable * PGLZ_V2_SUBTABLE_SIZE, 0, + PGLZ_V2_SUBTABLE_SIZE * sizeof(uint16)); + root_table[prefix] = PGLZ_V2_SUBTABLE_FLAG | subtable; + } + else if (root_table[prefix] & PGLZ_V2_SUBTABLE_FLAG) + subtable = root_table[prefix] & + ~PGLZ_V2_SUBTABLE_FLAG; + else + return false; + + first = (codes[sym] & ((1 << suffix_bits) - 1)) << + (PGLZ_V2_DECODE_EXTRA - suffix_bits); + count = 1 << (PGLZ_V2_DECODE_EXTRA - suffix_bits); + for (int i = 0; i < count; i++) + { + uint16 *slot = subtables + + subtable * PGLZ_V2_SUBTABLE_SIZE + first + i; + + if (*slot != 0) + return false; + *slot = entry; + } + } + } + + /* + * Use otherwise redundant root-table suffix bits to decode a second + * literal. The packed length includes the first code, the next literal's + * zero token bit, and the second code. Long codes still use the ordinary + * subtable path. + */ + for (int index = 0; index < (1 << PGLZ_V2_DECODE_BITS); index++) + { + uint32 entry = root_table[index]; + uint32 next_entry; + int first_length; + int available; + int next_index; + int second_length; + + if (entry == 0 || (entry & PGLZ_V2_SUBTABLE_FLAG) != 0) + continue; + first_length = (entry >> 8) & 0xff; + available = PGLZ_V2_DECODE_BITS - first_length - 1; + if (available <= 0 || ((index >> available) & 1) != 0) + continue; + + next_index = (index & ((1 << available) - 1)) << + (PGLZ_V2_DECODE_BITS - available); + next_entry = root_table[next_index]; + if (next_entry == 0 || + (next_entry & PGLZ_V2_SUBTABLE_FLAG) != 0) + continue; + second_length = (next_entry >> 8) & 0xff; + if (second_length > available) + continue; + + root_table[index] |= PGLZ_V2_LITERAL_PAIR_FLAG | + ((next_entry & 0xff) << PGLZ_V2_PAIR_SYMBOL_SHIFT) | + ((first_length + 1 + second_length) << + PGLZ_V2_PAIR_LENGTH_SHIFT); + } + return true; +} + +static inline void +pglz_copy_match(unsigned char **dest, int32 length, int32 offset, + unsigned char *destend) +{ + unsigned char *dp = *dest; + + /* + * Away from the output boundary, copy a fixed amount for common short + * matches. Bytes beyond the match are harmless because a later token + * overwrites them. Staging the copies in offset-sized-or-smaller chunks + * also produces the right repetition when source and destination overlap. + */ + if (length <= 18 && offset >= 8 && destend - dp >= 18) + { + uint64 word; + uint16 tail; + + memcpy(&word, dp - offset, sizeof(word)); + memcpy(dp, &word, sizeof(word)); + memcpy(&word, dp - offset + 8, sizeof(word)); + memcpy(dp + 8, &word, sizeof(word)); + memcpy(&tail, dp - offset + 16, sizeof(tail)); + memcpy(dp + 16, &tail, sizeof(tail)); + *dest = dp + length; + return; + } + if (length <= 32 && offset >= 16 && destend - dp >= 32) + { + uint64 word0; + uint64 word1; + + memcpy(&word0, dp - offset, sizeof(word0)); + memcpy(&word1, dp - offset + 8, sizeof(word1)); + memcpy(dp, &word0, sizeof(word0)); + memcpy(dp + 8, &word1, sizeof(word1)); + memcpy(&word0, dp - offset + 16, sizeof(word0)); + memcpy(&word1, dp - offset + 24, sizeof(word1)); + memcpy(dp + 16, &word0, sizeof(word0)); + memcpy(dp + 24, &word1, sizeof(word1)); + *dest = dp + length; + return; + } + if (length <= 64 && offset >= 32 && destend - dp >= 64) + { + memcpy(dp, dp - offset, 32); + memcpy(dp + 32, dp - offset + 32, 32); + *dest = dp + length; + return; + } + + /* Avoid a library call for other small, non-overlapping matches. */ + if (offset >= length && length <= 64) + { + while (length >= 8) + { + uint64 word; + + memcpy(&word, dp - offset, sizeof(word)); + memcpy(dp, &word, sizeof(word)); + dp += sizeof(word); + length -= sizeof(word); + } + if (length >= 4) + { + uint32 word; + + memcpy(&word, dp - offset, sizeof(word)); + memcpy(dp, &word, sizeof(word)); + dp += sizeof(word); + length -= sizeof(word); + } + if (length >= 2) + { + uint16 word; + + memcpy(&word, dp - offset, sizeof(word)); + memcpy(dp, &word, sizeof(word)); + dp += sizeof(word); + length -= sizeof(word); + } + if (length != 0) + { + *dp = *(dp - offset); + dp++; + } + *dest = dp; + return; + } + + while (offset < length) + { + memcpy(dp, dp - offset, offset); + length -= offset; + dp += offset; + offset += offset; + } + memcpy(dp, dp - offset, length); + *dest = dp + length; +} + +static inline bool +pglz_v3_get_varint(const unsigned char **ptr, const unsigned char *end, + uint32 *value) +{ + uint32 result = 0; + + for (int shift = 0; shift <= 28; shift += 7) + { + unsigned char byte; + + if (*ptr >= end) + return false; + byte = *(*ptr)++; + if (shift == 28 && (byte & 0xf0) != 0) + return false; + result |= (uint32) (byte & 0x7f) << shift; + if ((byte & 0x80) == 0) + { + *value = result; + return true; + } + } + return false; +} + +static inline void +pglz_copy_literals(unsigned char *dest, const unsigned char *source, + int32 length, unsigned char *destend, + const unsigned char *srcend) +{ + if (length > 0 && length <= 16 && destend - dest >= 16 && + srcend - source >= 16) + { + uint64 word0; + uint64 word1; + + memcpy(&word0, source, sizeof(word0)); + memcpy(&word1, source + 8, sizeof(word1)); + memcpy(dest, &word0, sizeof(word0)); + memcpy(dest + 8, &word1, sizeof(word1)); + } + else + memcpy(dest, source, length); +} + +static int32 +pglz_decompress_v3(const char *source, int32 slen, char *dest, + int32 rawsize, bool check_complete) +{ + const unsigned char *sp; + const unsigned char *srcend = (const unsigned char *) source + slen; + unsigned char *dp = (unsigned char *) dest; + unsigned char *destend = dp + rawsize; + + if (slen < (int32) (sizeof(pglz_v3_magic) + 1)) + return -1; + sp = (const unsigned char *) source + sizeof(pglz_v3_magic); + + while (dp < destend) + { + unsigned char header; + uint32 extra; + int32 literal_length; + int32 copy_length; + int32 match_length; + int32 match_offset; + + /* Decode the common sequence with boundary checks hoisted. */ + if (destend - dp >= 64 && srcend - sp >= 32) + { + header = *sp; + literal_length = header >> PGLZ_V3_LITERAL_SHIFT; + match_length = header & PGLZ_V3_MATCH_MASK; + if (literal_length != PGLZ_V3_LITERAL_MASK) + { + sp++; + pglz_copy_literals(dp, sp, literal_length, destend, srcend); + dp += literal_length; + sp += literal_length; + match_offset = (sp[0] | (sp[1] << 8)) + 1; + sp += 2; + if (match_offset > PGLZ_V2_WINDOW_SIZE || + match_offset > dp - (unsigned char *) dest) + return -1; + match_length += PGLZ_V2_MIN_MATCH; + if ((header & PGLZ_V3_MATCH_MASK) == PGLZ_V3_MATCH_MASK) + { + if (!pglz_v3_get_varint(&sp, srcend, &extra) || + extra > PGLZ_V2_MAX_MATCH - PGLZ_V2_MIN_MATCH - + PGLZ_V3_MATCH_MASK) + return -1; + match_length += extra; + if (check_complete && match_length > destend - dp) + return -1; + match_length = Min(match_length, destend - dp); + } + pglz_copy_match(&dp, match_length, match_offset, destend); + continue; + } + } + + if (sp >= srcend) + return -1; + header = *sp++; + literal_length = (header >> PGLZ_V3_LITERAL_SHIFT) & + PGLZ_V3_LITERAL_MASK; + if (literal_length == PGLZ_V3_LITERAL_MASK) + { + if (!pglz_v3_get_varint(&sp, srcend, &extra) || + extra > INT_MAX - PGLZ_V3_LITERAL_MASK) + return -1; + literal_length += extra; + } + if (check_complete && literal_length > destend - dp) + return -1; + + copy_length = Min(literal_length, destend - dp); + if (copy_length > srcend - sp) + return -1; + pglz_copy_literals(dp, sp, copy_length, destend, srcend); + dp += copy_length; + sp += copy_length; + if (copy_length != literal_length) + return (char *) dp - dest; + + if (sp == srcend) + { + if ((header & PGLZ_V3_MATCH_MASK) != 0 || dp != destend) + return -1; + return (char *) dp - dest; + } + if (dp == destend) + return check_complete ? -1 : (char *) dp - dest; + + if (srcend - sp < 2) + return -1; + match_offset = (sp[0] | (sp[1] << 8)) + 1; + sp += 2; + + match_length = PGLZ_V2_MIN_MATCH + + (header & PGLZ_V3_MATCH_MASK); + if ((header & PGLZ_V3_MATCH_MASK) == PGLZ_V3_MATCH_MASK) + { + if (!pglz_v3_get_varint(&sp, srcend, &extra) || + extra > PGLZ_V2_MAX_MATCH - PGLZ_V2_MIN_MATCH - + PGLZ_V3_MATCH_MASK) + return -1; + match_length += extra; + } + if (match_offset > PGLZ_V2_WINDOW_SIZE || + match_offset > dp - (unsigned char *) dest || + (check_complete && match_length > destend - dp)) + return -1; + match_length = Min(match_length, destend - dp); + pglz_copy_match(&dp, match_length, match_offset, destend); + } + + if (check_complete && sp != srcend) + return -1; + return (char *) dp - dest; +} + +static int32 +pglz_decompress_v2(const char *source, int32 slen, char *dest, + int32 rawsize, bool check_complete) +{ + const unsigned char *sp; + const unsigned char *srcend = (const unsigned char *) source + slen; + const unsigned char *bitmap; + unsigned char *dp = (unsigned char *) dest; + unsigned char *destend = dp + rawsize; + uint8 lengths[256] = {0}; + uint32 root_table[1 << PGLZ_V2_DECODE_BITS]; + uint16 subtables[PGLZ_V2_MAX_SUBTABLES * PGLZ_V2_SUBTABLE_SIZE]; + PGLZ_BitReader reader; + int nsymbols = 0; + int length_index = 0; + + if (slen < (int32) (sizeof(pglz_v2_magic) + PGLZ_V2_BITMAP_SIZE + 1)) + return -1; + sp = (const unsigned char *) source + sizeof(pglz_v2_magic); + bitmap = sp; + sp += PGLZ_V2_BITMAP_SIZE; + + for (int sym = 0; sym < 256; sym++) + { + if (bitmap[sym >> 3] & (1U << (sym & 7))) + nsymbols++; + } + if (nsymbols == 0 || srcend - sp < (nsymbols + 1) / 2) + return -1; + + for (int sym = 0; sym < 256; sym++) + { + uint8 length_byte; + + if ((bitmap[sym >> 3] & (1U << (sym & 7))) == 0) + continue; + length_byte = sp[length_index >> 1]; + if (length_index & 1) + lengths[sym] = length_byte >> 4; + else + lengths[sym] = length_byte & 0x0f; + length_index++; + } + if ((nsymbols & 1) != 0 && (sp[nsymbols >> 1] & 0xf0) != 0) + return -1; + sp += (nsymbols + 1) / 2; + if (!pglz_huffman_decode_tables(lengths, root_table, subtables)) + return -1; + + pglz_bitreader_init(&reader, sp, srcend); + while (dp < destend) + { + uint32 token_bits; + + token_bits = pglz_bitreader_peek_padded(&reader, + PGLZ_V2_HUFF_BITS + 1); + if ((token_bits >> PGLZ_V2_HUFF_BITS) == 0) + { + uint32 bits; + uint32 root_index; + uint32 entry; + int code_length; + + bits = token_bits & ((1 << PGLZ_V2_HUFF_BITS) - 1); + root_index = bits >> PGLZ_V2_DECODE_EXTRA; + entry = root_table[root_index]; + if (entry & PGLZ_V2_SUBTABLE_FLAG) + { + int subtable = entry & ~PGLZ_V2_SUBTABLE_FLAG; + + entry = subtables[subtable * PGLZ_V2_SUBTABLE_SIZE + + (bits & (PGLZ_V2_SUBTABLE_SIZE - 1))]; + } + else if ((entry & PGLZ_V2_LITERAL_PAIR_FLAG) != 0 && + dp + 1 < destend) + { + int pair_length = + (entry >> PGLZ_V2_PAIR_LENGTH_SHIFT) & 0x0f; + + if (!pglz_bitreader_drop(&reader, pair_length + 1)) + return -1; + *dp++ = entry & 0xff; + *dp++ = (entry >> PGLZ_V2_PAIR_SYMBOL_SHIFT) & 0xff; + continue; + } + code_length = (entry >> 8) & 0xff; + if (code_length == 0 || + !pglz_bitreader_drop(&reader, code_length + 1)) + return -1; + *dp++ = entry & 0xff; + } + else + { + uint32 value; + uint32 length_code; + int32 offset; + int32 length; + + if ((token_bits >> (PGLZ_V2_HUFF_BITS - 1)) & 1) + { + uint32 match_bits; + + match_bits = pglz_bitreader_peek_padded(&reader, 22); + value = (match_bits >> 4) & 0xffff; + length_code = match_bits & 0x0f; + if (!pglz_bitreader_drop(&reader, 22)) + return -1; + } + else + { + value = (token_bits >> 4) & 0xff; + length_code = token_bits & 0x0f; + if (!pglz_bitreader_drop(&reader, + PGLZ_V2_HUFF_BITS + 1)) + return -1; + } + offset = value + 1; + length = PGLZ_V2_MIN_MATCH + length_code; + if (length_code == 15) + { + uint32 first_byte; + uint32 second_byte; + uint32 third_byte; + uint32 extra; + + first_byte = pglz_bitreader_peek_padded(&reader, 8); + if (!pglz_bitreader_drop(&reader, 8)) + return -1; + extra = first_byte & 0x7f; + if (first_byte & 0x80) + { + second_byte = pglz_bitreader_peek_padded(&reader, 8); + if (!pglz_bitreader_drop(&reader, 8)) + return -1; + extra |= (second_byte & 0x7f) << 7; + if (second_byte & 0x80) + { + third_byte = pglz_bitreader_peek_padded(&reader, 8); + if (!pglz_bitreader_drop(&reader, 8)) + return -1; + extra |= (third_byte & 0x7f) << 14; + if (third_byte & 0x80) + return -1; + } + } + if (extra > PGLZ_V2_MAX_MATCH - + PGLZ_V2_MIN_MATCH - 15) + return -1; + length += extra; + } + + if (offset > PGLZ_V2_WINDOW_SIZE || + offset > dp - (unsigned char *) dest || + (check_complete && length > destend - dp)) + return -1; + length = Min(length, destend - dp); + pglz_copy_match(&dp, length, offset, destend); + } + } + + if (check_complete && + (reader.ptr != reader.end || reader.nbits >= 8 || + (reader.nbits > 0 && + (reader.bits & ((((uint64) 1) << reader.nbits) - 1)) != 0))) + return -1; + return (char *) dp - dest; +} + +int32 +pglz_decompress(const char *source, int32 slen, char *dest, + int32 rawsize, bool check_complete) +{ + if (slen >= (int32) sizeof(pglz_v3_magic) && + memcmp(source, pglz_v3_magic, sizeof(pglz_v3_magic)) == 0) + return pglz_decompress_v3(source, slen, dest, rawsize, + check_complete); + if (slen >= (int32) sizeof(pglz_v2_magic) && + memcmp(source, pglz_v2_magic, sizeof(pglz_v2_magic)) == 0) + return pglz_decompress_v2(source, slen, dest, rawsize, + check_complete); + return pglz_decompress_legacy(source, slen, dest, rawsize, + check_complete); +} + + /* ---------- * pglz_decompress - * @@ -688,9 +2534,9 @@ pglz_compress(const char *source, int32 slen, char *dest, * are extracting a slice typically can't apply this check. * ---------- */ -int32 -pglz_decompress(const char *source, int32 slen, char *dest, - int32 rawsize, bool check_complete) +static int32 +pglz_decompress_legacy(const char *source, int32 slen, char *dest, + int32 rawsize, bool check_complete) { const unsigned char *sp; const unsigned char *srcend; @@ -856,6 +2702,9 @@ pglz_decompress(const char *source, int32 slen, char *dest, int32 pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size) { + int64 legacy_size; + int64 v2_size; + int64 v3_size; int64 compressed_size; /* @@ -865,7 +2714,7 @@ pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size) * * Use int64 here to prevent overflow during calculation. */ - compressed_size = ((int64) rawsize * 9 + 7) / 8; + legacy_size = ((int64) rawsize * 9 + 7) / 8; /* * The above fails to account for a corner case: we could have compressed @@ -875,7 +2724,23 @@ pglz_maximum_compressed_size(int32 rawsize, int32 total_compressed_size) * in the compressed data don't cause a problem, since they should * represent more decompressed bytes than they occupy themselves.) */ - compressed_size += 2; + legacy_size += 2; + + /* + * Version 2 needs its Huffman header, and a literal can occupy at most + * one token bit plus PGLZ_V2_HUFF_BITS code bits. Six bytes of slop + * cover the largest offset and length encoding for a match that crosses + * the requested output boundary. + */ + v2_size = PGLZ_V2_MAX_HEADER + + (((int64) rawsize * (PGLZ_V2_HUFF_BITS + 1) + 7) / 8) + 6; + + /* + * A version 3 literal occupies one byte, with enough header slop for a + * length extension and a match that crosses the requested boundary. + */ + v3_size = (int64) rawsize + PGLZ_V3_MAX_HEADER; + compressed_size = Max(Max(legacy_size, v2_size), v3_size); /* * Maximum compressed size can't be larger than total compressed size. diff --git a/src/test/regress/expected/compression_pglz.out b/src/test/regress/expected/compression_pglz.out index 066a331..c59b470 100644 --- a/src/test/regress/expected/compression_pglz.out +++ b/src/test/regress/expected/compression_pglz.out @@ -11,6 +11,9 @@ CREATE FUNCTION test_pglz_compress(bytea) CREATE FUNCTION test_pglz_decompress(bytea, int4, bool) RETURNS bytea AS :'regresslib' LANGUAGE C STRICT; +CREATE FUNCTION test_pglz_maximum_compressed_size(int4, int4) + RETURNS int4 + AS :'regresslib' LANGUAGE C STRICT; -- Round-trip with pglz: compress then decompress. SELECT test_pglz_decompress(test_pglz_compress( decode(repeat('abcd', 100), 'escape')), 400, false) = @@ -38,6 +41,41 @@ ERROR: pglz_decompress failed SELECT test_pglz_decompress(test_pglz_compress( decode(repeat('abcd', 100), 'escape')), 100, true); ERROR: pglz_decompress failed +-- A partial decompression only needs enough of a long literal run to fill +-- the requested output slice. The repeated 0..255 block enters the modern +-- compressor and produces an initial 256-byte literal run. +WITH input AS +( + SELECT decode(repeat(string_agg(lpad(to_hex(g), 2, '0'), '' ORDER BY g), 128), + 'hex') AS data + FROM generate_series(0, 255) g +), compressed AS +( + SELECT data, test_pglz_compress(data) AS data_compressed + FROM input +) +SELECT get_byte(data_compressed, 6) = ascii('3') AS version_3, + test_pglz_decompress(data_compressed, length(data), true) = data + AS roundtrip_ok, + test_pglz_decompress( + substring(data_compressed FOR + test_pglz_maximum_compressed_size(1, length(data_compressed))), + 1, false) = substring(data FOR 1) AS slice_ok +FROM compressed; + version_3 | roundtrip_ok | slice_ok +-----------+--------------+---------- + t | t | t +(1 row) + +-- Corrupted version 3 data: marker without a sequence header. +SELECT test_pglz_decompress('\x01000050474c33'::bytea, 1, false); +ERROR: pglz_decompress failed +-- Corrupted version 3 data: literal length without its extension. +SELECT test_pglz_decompress('\x01000050474c33f0'::bytea, 1, false); +ERROR: pglz_decompress failed +-- Corrupted version 3 data: match offset exceeds the output written. +SELECT test_pglz_decompress('\x01000050474c33000000'::bytea, 6, true); +ERROR: pglz_decompress failed -- Corrupted compressed data. Set control bit with read of a match tag, -- no data follows. SELECT length(test_pglz_decompress('\x01'::bytea, 1024, false)) AS ctrl_only_len; @@ -75,3 +113,4 @@ ERROR: pglz_decompress failed -- Clean up DROP FUNCTION test_pglz_compress; DROP FUNCTION test_pglz_decompress; +DROP FUNCTION test_pglz_maximum_compressed_size; diff --git a/src/test/regress/expected/strings.out b/src/test/regress/expected/strings.out index a49b75f..1d2c7c5 100644 --- a/src/test/regress/expected/strings.out +++ b/src/test/regress/expected/strings.out @@ -2190,7 +2190,9 @@ TRUNCATE toasttest; -- test with external compressed data (default). ALTER TABLE toasttest ALTER COLUMN f1 SET STORAGE EXTENDED; ALTER TABLE toasttest ALTER COLUMN f2 SET STORAGE EXTENDED; -INSERT INTO toasttest values(repeat('1234', 10240), NULL); +INSERT INTO toasttest + SELECT repeat('1234', 256) || string_agg(md5(g::text), ''), NULL + FROM generate_series(1, 1280) g; -- There should be one value in the toast relation. SELECT substr(f1, 5, 10) AS f1_data, substr(f2, 5, 10) AS f2_data FROM toasttest; diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 6ee6689..3c61320 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -1492,3 +1492,18 @@ test_pglz_decompress(PG_FUNCTION_ARGS) SET_VARSIZE(result, dlen + VARHDRSZ); PG_RETURN_BYTEA_P(result); } + +/* Expose the compressed-prefix bound for slice decompression tests. */ +PG_FUNCTION_INFO_V1(test_pglz_maximum_compressed_size); +Datum +test_pglz_maximum_compressed_size(PG_FUNCTION_ARGS) +{ + int32 rawsize = PG_GETARG_INT32(0); + int32 total_compressed_size = PG_GETARG_INT32(1); + + if (rawsize < 0 || total_compressed_size < 0) + elog(ERROR, "sizes must not be negative"); + + PG_RETURN_INT32(pglz_maximum_compressed_size(rawsize, + total_compressed_size)); +} diff --git a/src/test/regress/sql/compression_pglz.sql b/src/test/regress/sql/compression_pglz.sql index dbd37f7..7c0155f 100644 --- a/src/test/regress/sql/compression_pglz.sql +++ b/src/test/regress/sql/compression_pglz.sql @@ -14,6 +14,9 @@ CREATE FUNCTION test_pglz_compress(bytea) CREATE FUNCTION test_pglz_decompress(bytea, int4, bool) RETURNS bytea AS :'regresslib' LANGUAGE C STRICT; +CREATE FUNCTION test_pglz_maximum_compressed_size(int4, int4) + RETURNS int4 + AS :'regresslib' LANGUAGE C STRICT; -- Round-trip with pglz: compress then decompress. SELECT test_pglz_decompress(test_pglz_compress( @@ -33,6 +36,37 @@ SELECT test_pglz_decompress(test_pglz_compress( SELECT test_pglz_decompress(test_pglz_compress( decode(repeat('abcd', 100), 'escape')), 100, true); +-- A partial decompression only needs enough of a long literal run to fill +-- the requested output slice. The repeated 0..255 block enters the modern +-- compressor and produces an initial 256-byte literal run. +WITH input AS +( + SELECT decode(repeat(string_agg(lpad(to_hex(g), 2, '0'), '' ORDER BY g), 128), + 'hex') AS data + FROM generate_series(0, 255) g +), compressed AS +( + SELECT data, test_pglz_compress(data) AS data_compressed + FROM input +) +SELECT get_byte(data_compressed, 6) = ascii('3') AS version_3, + test_pglz_decompress(data_compressed, length(data), true) = data + AS roundtrip_ok, + test_pglz_decompress( + substring(data_compressed FOR + test_pglz_maximum_compressed_size(1, length(data_compressed))), + 1, false) = substring(data FOR 1) AS slice_ok +FROM compressed; + +-- Corrupted version 3 data: marker without a sequence header. +SELECT test_pglz_decompress('\x01000050474c33'::bytea, 1, false); + +-- Corrupted version 3 data: literal length without its extension. +SELECT test_pglz_decompress('\x01000050474c33f0'::bytea, 1, false); + +-- Corrupted version 3 data: match offset exceeds the output written. +SELECT test_pglz_decompress('\x01000050474c33000000'::bytea, 6, true); + -- Corrupted compressed data. Set control bit with read of a match tag, -- no data follows. SELECT length(test_pglz_decompress('\x01'::bytea, 1024, false)) AS ctrl_only_len; @@ -61,3 +95,4 @@ SELECT test_pglz_decompress('\x010300'::bytea, 1024, true); -- Clean up DROP FUNCTION test_pglz_compress; DROP FUNCTION test_pglz_decompress; +DROP FUNCTION test_pglz_maximum_compressed_size; diff --git a/src/test/regress/sql/strings.sql b/src/test/regress/sql/strings.sql index 5ae0e7d..70c1716 100644 --- a/src/test/regress/sql/strings.sql +++ b/src/test/regress/sql/strings.sql @@ -694,7 +694,9 @@ TRUNCATE toasttest; -- test with external compressed data (default). ALTER TABLE toasttest ALTER COLUMN f1 SET STORAGE EXTENDED; ALTER TABLE toasttest ALTER COLUMN f2 SET STORAGE EXTENDED; -INSERT INTO toasttest values(repeat('1234', 10240), NULL); +INSERT INTO toasttest + SELECT repeat('1234', 256) || string_agg(md5(g::text), ''), NULL + FROM generate_series(1, 1280) g; -- There should be one value in the toast relation. SELECT substr(f1, 5, 10) AS f1_data, substr(f2, 5, 10) AS f2_data FROM toasttest;