[PATCH 0/3] btrfs: Add zstd support to btrfs
Hi all, This patch set imports the upstream kernel zstd library, patches it to work in grub, adds zstd support to the btrfs module, and adds a test case. I've separated the importing and patching of the upstream kernel zstd library for ease of review. Thanks to David Sterba for starting this project! I started my patch from where he left off. Best, Nick Terrell Nick Terrell (3): Import kernel zstd Patch the kernel zstd Add btrfs zstd support Makefile.util.def |8 +- grub-core/Makefile.core.def | 10 +- grub-core/fs/btrfs.c| 85 +- grub-core/lib/zstd/bitstream.h | 380 ++ grub-core/lib/zstd/decompress.c | 2494 +++ grub-core/lib/zstd/entropy_common.c | 243 grub-core/lib/zstd/error_private.h | 52 + grub-core/lib/zstd/fse.h| 575 grub-core/lib/zstd/fse_decompress.c | 329 + grub-core/lib/zstd/huf.h| 212 +++ grub-core/lib/zstd/huf_decompress.c | 957 ++ grub-core/lib/zstd/kerncompat.h | 69 + grub-core/lib/zstd/mem.h| 155 +++ grub-core/lib/zstd/xxhash.c | 482 +++ grub-core/lib/zstd/xxhash.h | 236 grub-core/lib/zstd/zstd.h | 1157 grub-core/lib/zstd/zstd_common.c| 75 ++ grub-core/lib/zstd/zstd_internal.h | 265 tests/btrfs_test.in |1 + tests/util/grub-fs-tester.in|4 +- 20 files changed, 7783 insertions(+), 6 deletions(-) create mode 100644 grub-core/lib/zstd/bitstream.h create mode 100644 grub-core/lib/zstd/decompress.c create mode 100644 grub-core/lib/zstd/entropy_common.c create mode 100644 grub-core/lib/zstd/error_private.h create mode 100644 grub-core/lib/zstd/fse.h create mode 100644 grub-core/lib/zstd/fse_decompress.c create mode 100644 grub-core/lib/zstd/huf.h create mode 100644 grub-core/lib/zstd/huf_decompress.c create mode 100644 grub-core/lib/zstd/kerncompat.h create mode 100644 grub-core/lib/zstd/mem.h create mode 100644 grub-core/lib/zstd/xxhash.c create mode 100644 grub-core/lib/zstd/xxhash.h create mode 100644 grub-core/lib/zstd/zstd.h create mode 100644 grub-core/lib/zstd/zstd_common.c create mode 100644 grub-core/lib/zstd/zstd_internal.h -- 2.16.2 ___ Grub-devel mailing list Grub-devel@gnu.org https://lists.gnu.org/mailman/listinfo/grub-devel
[PATCH 3/3] btrfs: Add zstd support to btrfs
Adds zstd support to the btrfs module. I'm not sure that my changes to the
Makefiles are correct, please let me know if I need to do something
differently.
Tested on Ubuntu-18.04 with a btrfs /boot partition with and without zstd
compression. A test case was also added to the test suite that fails before
the patch, and passes after.
Signed-off-by: Nick Terrell
---
Makefile.util.def| 8 -
grub-core/Makefile.core.def | 10 --
grub-core/fs/btrfs.c | 85 +++-
tests/btrfs_test.in | 1 +
tests/util/grub-fs-tester.in | 4 +--
5 files changed, 102 insertions(+), 6 deletions(-)
diff --git a/Makefile.util.def b/Makefile.util.def
index 3180ac880..b987dc637 100644
--- a/Makefile.util.def
+++ b/Makefile.util.def
@@ -54,7 +54,7 @@ library = {
library = {
name = libgrubmods.a;
cflags = '-fno-builtin -Wno-undef';
- cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo
-I$(srcdir)/grub-core/lib/xzembed -DMINILZO_HAVE_CONFIG_H';
+ cppflags = '-I$(top_srcdir)/grub-core/lib/minilzo
-I$(srcdir)/grub-core/lib/xzembed -I$(top_srcdir)/grub-core/lib/zstd
-DMINILZO_HAVE_CONFIG_H';
common_nodist = grub_script.tab.c;
common_nodist = grub_script.yy.c;
@@ -165,6 +165,12 @@ library = {
common = grub-core/lib/xzembed/xz_dec_bcj.c;
common = grub-core/lib/xzembed/xz_dec_lzma2.c;
common = grub-core/lib/xzembed/xz_dec_stream.c;
+ common = grub-core/lib/zstd/zstd_common.c;
+ common = grub-core/lib/zstd/huf_decompress.c;
+ common = grub-core/lib/zstd/fse_decompress.c;
+ common = grub-core/lib/zstd/entropy_common.c;
+ common = grub-core/lib/zstd/decompress.c;
+ common = grub-core/lib/zstd/xxhash.c;
};
program = {
diff --git a/grub-core/Makefile.core.def b/grub-core/Makefile.core.def
index 9590e87d9..c24bf9147 100644
--- a/grub-core/Makefile.core.def
+++ b/grub-core/Makefile.core.def
@@ -404,7 +404,7 @@ image = {
i386_pc = boot/i386/pc/boot.S;
cppflags = '-DHYBRID_BOOT=1';
-
+
i386_pc_ldflags = '$(TARGET_IMG_LDFLAGS)';
i386_pc_ldflags = '$(TARGET_IMG_BASE_LDOPT),0x7C00';
@@ -1264,8 +1264,14 @@ module = {
name = btrfs;
common = fs/btrfs.c;
common = lib/crc.c;
+ common = lib/zstd/zstd_common.c;
+ common = lib/zstd/huf_decompress.c;
+ common = lib/zstd/fse_decompress.c;
+ common = lib/zstd/entropy_common.c;
+ common = lib/zstd/decompress.c;
+ common = lib/zstd/xxhash.c;
cflags = '$(CFLAGS_POSIX) -Wno-undef';
- cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo
-DMINILZO_HAVE_CONFIG_H';
+ cppflags = '-I$(srcdir)/lib/posix_wrap -I$(srcdir)/lib/minilzo
-I$(srcdir)/lib/zstd -DMINILZO_HAVE_CONFIG_H';
};
module = {
diff --git a/grub-core/fs/btrfs.c b/grub-core/fs/btrfs.c
index be195448d..89ed4884e 100644
--- a/grub-core/fs/btrfs.c
+++ b/grub-core/fs/btrfs.c
@@ -27,6 +27,7 @@
#include
#include
#include
+#include
#include
#include
@@ -45,6 +46,9 @@ GRUB_MOD_LICENSE ("GPLv3+");
#define GRUB_BTRFS_LZO_BLOCK_MAX_CSIZE (GRUB_BTRFS_LZO_BLOCK_SIZE + \
(GRUB_BTRFS_LZO_BLOCK_SIZE / 16) + 64 + 3)
+#define ZSTD_BTRFS_MAX_WINDOWLOG 17
+#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
+
typedef grub_uint8_t grub_btrfs_checksum_t[0x20];
typedef grub_uint16_t grub_btrfs_uuid_t[8];
@@ -212,6 +216,7 @@ struct grub_btrfs_extent_data
#define GRUB_BTRFS_COMPRESSION_NONE 0
#define GRUB_BTRFS_COMPRESSION_ZLIB 1
#define GRUB_BTRFS_COMPRESSION_LZO 2
+#define GRUB_BTRFS_COMPRESSION_ZSTD 3
#define GRUB_BTRFS_OBJECT_ID_CHUNK 0x100
@@ -912,6 +917,70 @@ grub_btrfs_read_inode (struct grub_btrfs_data *data,
return grub_btrfs_read_logical (data, elemaddr, inode, sizeof (*inode), 0);
}
+static grub_ssize_t
+grub_btrfs_zstd_decompress(char *ibuf, grub_size_t isize, grub_off_t off,
+ char *obuf, grub_size_t osize)
+{
+ void *allocated = NULL;
+ char *otmpbuf = obuf;
+ grub_size_t otmpsize = osize;
+ void *wmem = NULL;
+ const grub_size_t wmem_size = ZSTD_DCtxWorkspaceBound ();
+ ZSTD_DCtx *dctx;
+ grub_size_t zstd_ret;
+ grub_ssize_t ret = -1;
+
+ /* Zstd will fail if it can't fit the entire output in the destination
+* buffer, so if osize isn't large enough, allocate a temporary buffer.
+*/
+ if (otmpsize < ZSTD_BTRFS_MAX_INPUT) {
+ allocated = grub_malloc (ZSTD_BTRFS_MAX_INPUT);
+ if (!allocated) {
+ grub_dprintf ("zstd", "outtmpbuf allocation failed\n");
+ goto out;
+ }
+ otmpbuf = (char*)allocated;
+ otmpsize = ZSTD_BTRFS_MAX_INPUT;
+ }
+
+ /* Allocate space for, and initialize, the ZSTD_DCtx. */
+ wmem = grub_malloc (wmem_size);
+ if (!wmem) {
+ grub_dprintf ("zstd", "wmem allocation failed\n");
+ goto out;
+ }
+ dctx = ZSTD_initDCtx (wmem, wmem_size);
+
+ /* Ge
[PATCH 2/3] btrfs: Patch the kernel zstd
Applies patches to the upstream kernel zstd needed to compile in grub.
* Replace kernel headers with "kerncompat.h".
* Replace the unaligned memory access functions.
* Add fallthrough statements for newer gcc versions.
* Rename a variable from "byte" because of a conflicting typedef.
* Remove the module code.
* Switch the memcpy() in ZSTD_copy8() to __builtin_memcpy().
Signed-off-by: Nick Terrell
---
grub-core/lib/zstd/bitstream.h | 6
grub-core/lib/zstd/decompress.c | 56 ++
grub-core/lib/zstd/error_private.h | 3 +-
grub-core/lib/zstd/fse.h| 2 +-
grub-core/lib/zstd/fse_decompress.c | 3 --
grub-core/lib/zstd/huf.h| 2 +-
grub-core/lib/zstd/huf_decompress.c | 3 --
grub-core/lib/zstd/kerncompat.h | 69 +
grub-core/lib/zstd/mem.h| 42 --
grub-core/lib/zstd/xxhash.c | 22 ++--
grub-core/lib/zstd/xxhash.h | 2 +-
grub-core/lib/zstd/zstd.h | 2 +-
grub-core/lib/zstd/zstd_common.c| 2 +-
grub-core/lib/zstd/zstd_internal.h | 12 ---
14 files changed, 124 insertions(+), 102 deletions(-)
create mode 100644 grub-core/lib/zstd/kerncompat.h
diff --git a/grub-core/lib/zstd/bitstream.h b/grub-core/lib/zstd/bitstream.h
index a826b99e1..92a6c3fb9 100644
--- a/grub-core/lib/zstd/bitstream.h
+++ b/grub-core/lib/zstd/bitstream.h
@@ -51,6 +51,7 @@
* Dependencies
**/
#include "error_private.h" /* error codes and messages */
+#include "kerncompat.h"
#include "mem.h" /* unaligned access routines */
/*=
@@ -259,10 +260,15 @@ ZSTD_STATIC size_t BIT_initDStream(BIT_DStream_t *bitD,
const void *srcBuffer, s
bitD->bitContainer = *(const BYTE *)(bitD->start);
switch (srcSize) {
case 7: bitD->bitContainer += (size_t)(((const BYTE
*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer) * 8 - 16);
+ /* fallthrough */
case 6: bitD->bitContainer += (size_t)(((const BYTE
*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer) * 8 - 24);
+ /* fallthrough */
case 5: bitD->bitContainer += (size_t)(((const BYTE
*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer) * 8 - 32);
+ /* fallthrough */
case 4: bitD->bitContainer += (size_t)(((const BYTE
*)(srcBuffer))[3]) << 24;
+ /* fallthrough */
case 3: bitD->bitContainer += (size_t)(((const BYTE
*)(srcBuffer))[2]) << 16;
+ /* fallthrough */
case 2: bitD->bitContainer += (size_t)(((const BYTE
*)(srcBuffer))[1]) << 8;
default:;
}
diff --git a/grub-core/lib/zstd/decompress.c b/grub-core/lib/zstd/decompress.c
index b17846725..e7af8df09 100644
--- a/grub-core/lib/zstd/decompress.c
+++ b/grub-core/lib/zstd/decompress.c
@@ -33,9 +33,6 @@
#include "huf.h"
#include "mem.h" /* low level memory routines */
#include "zstd_internal.h"
-#include
-#include
-#include /* memcpy, memmove, memset */
#define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
@@ -123,6 +120,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx *dctx)
return 0;
}
+static
ZSTD_DCtx *ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
{
ZSTD_DCtx *dctx;
@@ -391,6 +389,7 @@ typedef struct {
/*! ZSTD_getcBlockSize() :
* Provides the size of compressed block from block header `src` */
+static
size_t ZSTD_getcBlockSize(const void *src, size_t srcSize, blockProperties_t
*bpPtr)
{
if (srcSize < ZSTD_blockHeaderSize)
@@ -429,6 +428,7 @@ static size_t ZSTD_setRleBlock(void *dst, size_t
dstCapacity, const void *src, s
/*! ZSTD_decodeLiteralsBlock() :
@return : nb of bytes read from src (< srcSize ) */
+static
size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx *dctx, const void *src, size_t
srcSize) /* note : srcSize < BLOCKSIZE */
{
if (srcSize < MIN_CBLOCK_SIZE)
@@ -791,6 +791,7 @@ static size_t ZSTD_buildSeqTable(FSE_DTable *DTableSpace,
const FSE_DTable **DTa
}
}
+static
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx *dctx, int *nbSeqPtr, const void *src,
size_t srcSize)
{
const BYTE *const istart = (const BYTE *const)src;
@@ -1494,11 +1495,12 @@ size_t ZSTD_insertBlock(ZSTD_DCtx *dctx, const void
*blockStart, size_t blockSiz
return blockSize;
}
-size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE byte, size_t
length)
+static
+size_t ZSTD_generateNxBytes(void *dst, size_t dstCapacity, BYTE value, size_t
length)
{
if (length > dstCapacity)
return ERROR(dstSize_tooSmall);
- memset(dst, byte, length);
+ memset(dst, value, length);
return length;
}
@@ -1768,6 +1770,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx *dctx, void
*dst, size_t dstCapacity, c
r
