This is a WIP patchset to further improve the resource manager. The
configure bits are the one that need more work: How do we detect zlib
(and the required library to link to) on the host system?
- Andreas
From 30e67850175e5dd6dffa7488a032bbbe14067d10 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Sun, 1 Jun 2025 22:38:28 +0200
Subject: [PATCH WIP 01/10] ffbuild/bin2c: Use zlib directly instead of gzip
Perform zlib compression directly in bin2c.c instead of
using gzip for compression, followed by bin2c to
put the file in the binary. This has the advantage of
avoiding intermediate files and also saves a few bytes
(the gzip header is longer than the zlib header).
It also avoids the hypothetical problem of the build system
supporting gzip with the zlib version used at runtime
not supporting gzip.
But the real advantages will become apparent in future commits.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
.gitignore | 1 -
ffbuild/bin2c.c | 154 ++++++++++++++++++++++++++++++---
ffbuild/common.mak | 44 ++--------
fftools/resources/.gitignore | 3 -
fftools/resources/Makefile | 2 +-
fftools/resources/resman.c | 6 +-
libavfilter/cuda/load_helper.c | 2 +-
7 files changed, 151 insertions(+), 61 deletions(-)
diff --git a/.gitignore b/.gitignore
index 59c89da5e0..901b47e76b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,7 +25,6 @@
*.metallib.c
*.ptx
*.ptx.c
-*.ptx.gz
*_g
\#*
.\#*
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index 91f2d81e5b..a3e49f0a11 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -20,48 +20,174 @@
* DEALINGS IN THE SOFTWARE.
*/
+#include "config.h"
+
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
#include <string.h>
#include <stdio.h>
+#include "libavutil/attributes.h"
+#include "libavutil/macros.h"
+
+#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
+#include <zlib.h>
+
+#define MAX_BUF_SIZE UINT32_MAX
+
+static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep,
+ FILE *input)
+{
+ z_stream zstream = { 0 };
+ int ret, zret = deflateInit(&zstream, 9);
+
+ if (zret != Z_OK)
+ return -1;
+
+ uint32_t buffer_size = 0, compressed_size = 0;
+ unsigned char *compressed_data = NULL;
+ int flush = Z_NO_FLUSH;
+
+ do {
+ unsigned char tmp[4096];
+ size_t read = fread(tmp, 1, sizeof(tmp), input);
+ if (read < sizeof(tmp))
+ flush = Z_FINISH;
+
+ zstream.next_in = tmp;
+ zstream.avail_in = read;
+
+ do {
+ if (compressed_size >= buffer_size) {
+ if (buffer_size == MAX_BUF_SIZE) {
+ ret = -1;
+ goto fail;
+ }
+
+ buffer_size = buffer_size ? FFMIN(compressed_size * 2ll, MAX_BUF_SIZE) : 4096;
+ void *tmp_ptr = realloc(compressed_data, buffer_size);
+ if (!tmp_ptr) {
+ ret = -1;
+ goto fail;
+ }
+ compressed_data = tmp_ptr;
+ }
+
+ zstream.next_out = compressed_data + compressed_size;
+ zstream.avail_out = buffer_size - compressed_size;
+
+ zret = deflate(&zstream, flush);
+ compressed_size = buffer_size - zstream.avail_out;
+ if (zret == Z_STREAM_END)
+ break;
+ if (zret != Z_OK) {
+ ret = -1;
+ goto fail;
+ }
+ } while (zstream.avail_in > 0 || zstream.avail_out == 0);
+ } while (flush != Z_FINISH);
+
+ deflateEnd(&zstream);
+ *compressed_datap = compressed_data;
+ *compressed_sizep = compressed_size;
+
+ return 0;
+fail:
+ free(compressed_data);
+ deflateEnd(&zstream);
+
+ return ret;
+}
+
+static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
+{
+ unsigned char *compressed_data;
+ uint32_t compressed_size;
+
+ int err = read_file_and_compress(&compressed_data, &compressed_size, input);
+ if (err)
+ return err;
+
+ *compressed_sizep = compressed_size;
+
+ for (unsigned i = 0; i < compressed_size; ++i)
+ fprintf(output, "0x%02x, ", compressed_data[i]);
+
+ free(compressed_data);
+
+ return 0;
+}
+#endif
+
int main(int argc, char **argv)
{
const char *name;
FILE *input, *output;
unsigned int length = 0;
unsigned char data;
+ av_unused int compression = 0;
+ int arg_idx = 1;
- if (argc < 3 || argc > 4)
+ if (argc < 3)
return 1;
- input = fopen(argv[1], "rb");
+ if (!strcmp(argv[arg_idx], "--compress")) {
+#if !CONFIG_PTX_COMPRESSION && !CONFIG_RESOURCE_COMPRESSION
+ fprintf(stderr, "Compression unsupported in this configuration. "
+ "This is a bug. Please report it.\n");
+ return -1;
+#endif
+ compression = 1;
+ ++arg_idx;
+ }
+
+ if (argc - arg_idx > 3)
+ return 1;
+
+ char *input_name = argv[arg_idx++];
+ input = fopen(input_name, "rb");
if (!input)
return -1;
- output = fopen(argv[2], "wb");
+ output = fopen(argv[arg_idx++], "wb");
if (!output) {
fclose(input);
return -1;
}
- if (argc == 4) {
- name = argv[3];
+ if (arg_idx < argc) {
+ name = argv[arg_idx++];
} else {
- size_t arglen = strlen(argv[1]);
- name = argv[1];
+ size_t arglen = strlen(input_name);
+ name = input_name;
for (int i = 0; i < arglen; i++) {
- if (argv[1][i] == '.')
- argv[1][i] = '_';
- else if (argv[1][i] == '/')
- name = &argv[1][i+1];
+ if (input_name[i] == '.')
+ input_name[i] = '_';
+ else if (input_name[i] == '/')
+ name = &input_name[i+1];
}
}
fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
- while (fread(&data, 1, 1, input) > 0) {
- fprintf(output, "0x%02x, ", data);
- length++;
+#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
+ if (compression) {
+ int err = handle_compressed_file(input, output, &length);
+ if (err) {
+ fclose(input);
+ fclose(output);
+ return err;
+ }
+ } else
+#endif
+ {
+ while (fread(&data, 1, 1, input) > 0) {
+ fprintf(output, "0x%02x, ", data);
+ length++;
+ }
}
fprintf(output, "0x00 };\n");
diff --git a/ffbuild/common.mak b/ffbuild/common.mak
index ddf48923ea..b830f9862b 100644
--- a/ffbuild/common.mak
+++ b/ffbuild/common.mak
@@ -113,7 +113,7 @@ COMPILE_LASX = $(call COMPILE,CC,LASXFLAGS)
$(Q)echo '#include "$*.h"' >$@
$(BIN2CEXE): ffbuild/bin2c_host.o
- $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS)
+ $(HOSTLD) $(HOSTLDFLAGS) $(HOSTLD_O) $^ $(HOSTEXTRALIBS) $(if $(CONFIG_PTX_COMPRESSION)$(CONFIG_RESOURCE_COMPRESSION),-lz)
%.metal.air: %.metal
$(METALCC) $< -o $@
@@ -127,17 +127,8 @@ $(BIN2CEXE): ffbuild/bin2c_host.o
%.ptx: %.cu $(SRC_PATH)/compat/cuda/cuda_runtime.h
$(COMPILE_NVCC)
-ifdef CONFIG_PTX_COMPRESSION
-%.ptx.gz: TAG = GZIP
-%.ptx.gz: %.ptx
- $(M)gzip -nc9 $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) >$@
-
-%.ptx.c: %.ptx.gz $(BIN2CEXE)
- $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
-else
%.ptx.c: %.ptx $(BIN2CEXE)
- $(BIN2C) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
-endif
+ $(BIN2C) $(if $(CONFIG_PTX_COMPRESSION),--compress) $(patsubst $(SRC_PATH)/%,$(SRC_LINK)/%,$<) $@ $(subst .,_,$(basename $(notdir $@)))
# 1) Preprocess CSS to a minified version
%.css.min: TAG = SED
@@ -148,36 +139,13 @@ endif
| sed 's/^ //; s/ $$//' \
> $@
-ifdef CONFIG_RESOURCE_COMPRESSION
-
-# 2) Gzip the minified CSS
-%.css.min.gz: TAG = GZIP
-%.css.min.gz: %.css.min
- $(M)gzip -nc9 $< > $@
-
-# 3) Convert the gzipped CSS to a .c array
-%.css.c: %.css.min.gz $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-
-# 4) Gzip the HTML file (no minification needed)
-%.html.gz: TAG = GZIP
-%.html.gz: %.html
- $(M)gzip -nc9 $< > $@
-
-# 5) Convert the gzipped HTML to a .c array
-%.html.c: %.html.gz $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-
-else # NO COMPRESSION
-
# 2) Convert the minified CSS to a .c array
%.css.c: %.css.min $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
+ $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@)))
# 3) Convert the plain HTML to a .c array
%.html.c: %.html $(BIN2CEXE)
- $(BIN2C) $< $@ $(subst .,_,$(basename $(notdir $@)))
-endif
+ $(BIN2C) $(if $(CONFIG_RESOURCE_COMPRESSION),--compress) $< $@ $(subst .,_,$(basename $(notdir $@)))
clean::
$(RM) $(BIN2CEXE) $(CLEANSUFFIXES:%=ffbuild/%)
@@ -232,7 +200,7 @@ PTXOBJS = $(filter %.ptx.o,$(OBJS))
RESOURCEOBJS = $(filter %.css.o %.html.o,$(OBJS))
$(HOBJS): CCFLAGS += $(CFLAGS_HEADERS)
checkheaders: $(HOBJS)
-.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=.gz) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:%.css.o=%.css.min.gz) $(RESOURCEOBJS:%.html.o=%.html.gz) $(RESOURCEOBJS:.o=)
+.SECONDARY: $(HOBJS:.o=.c) $(PTXOBJS:.o=.c) $(PTXOBJS:.o=) $(RESOURCEOBJS:.o=.c) $(RESOURCEOBJS:%.css.o=%.css.min) $(RESOURCEOBJS:.o=)
alltools: $(TOOLS)
@@ -252,7 +220,7 @@ $(TOOLOBJS): | tools
OUTDIRS := $(OUTDIRS) $(dir $(OBJS) $(HOBJS) $(HOSTOBJS) $(SHLIBOBJS) $(STLIBOBJS) $(TESTOBJS))
-CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.gz *.ptx.c *.ver *.version *.html.gz *.html.c *.css.gz *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
+CLEANSUFFIXES = *.d *.gcda *.gcno *.h.c *.ho *.map *.o *.objs *.pc *.ptx *.ptx.c *.ver *.version *.html.c *.css.c *$(DEFAULT_X86ASMD).asm *~ *.ilk *.pdb
LIBSUFFIXES = *.a *.lib *.so *.so.* *.dylib *.dll *.def *.dll.a
define RULES
diff --git a/fftools/resources/.gitignore b/fftools/resources/.gitignore
index bda2c59a1c..65febdf13b 100644
--- a/fftools/resources/.gitignore
+++ b/fftools/resources/.gitignore
@@ -1,6 +1,3 @@
*.html.c
*.css.c
-*.html.gz
-*.css.gz
*.min
-*.min.gz
diff --git a/fftools/resources/Makefile b/fftools/resources/Makefile
index 8579a52678..aeb631f206 100644
--- a/fftools/resources/Makefile
+++ b/fftools/resources/Makefile
@@ -5,7 +5,7 @@ vpath %.html $(SRC_PATH)
vpath %.css $(SRC_PATH)
# Uncomment to prevent deletion during build
-#.PRECIOUS: %.css.c %.css.min %.css.gz %.css.min.gz %.html.gz %.html.c
+#.PRECIOUS: %.css.c %.css.min %.html.c
OBJS-resman += \
fftools/resources/resman.o \
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index aa53e96bf4..fdf73c36d5 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
+static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
{
z_stream strm;
unsigned chunk = 65534;
@@ -83,7 +83,7 @@ static int decompress_gzip(ResourceManagerContext *ctx, uint8_t *in, unsigned in
}
// 15 + 16 tells zlib to detect GZIP or zlib automatically
- ret = inflateInit2(&strm, 15 + 16);
+ ret = inflateInit(&strm);
if (ret != Z_OK) {
av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
av_free(buf);
@@ -156,7 +156,7 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
size_t out_len;
- int ret = decompress_gzip(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
+ int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index b049ec7130..7c62793077 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -43,7 +43,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
uint64_t buf_size;
int ret;
- if (inflateInit2(&stream, 32 + 15) != Z_OK) {
+ if (inflateInit(&stream) != Z_OK) {
av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg);
return AVERROR(ENOSYS);
}
--
2.45.2
From c7f297f99bc68abcc535f29fcced8a81e668728f Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 02:46:58 +0200
Subject: [PATCH WIP 02/10] fftools/resources/resman: Don't mix FFmpeg and zlib
errors
Also remove some always-false checks.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
fftools/resources/resman.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index fdf73c36d5..f910535750 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -100,7 +100,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
inflateEnd(&strm);
av_free(buf);
- return (ret == Z_STREAM_END) ? Z_OK : ((ret == Z_OK) ? Z_BUF_ERROR : ret);
+ return AVERROR_EXTERNAL;
}
if (strm.avail_out == 0) {
@@ -113,7 +113,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
inflateEnd(&strm);
*out = (char *)buf;
- return Z_OK;
+ return 0;
}
#endif
--
2.45.2
From c4cdbee4002c1109d576a6e140f07b2bac3f9b81 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 02:53:24 +0200
Subject: [PATCH WIP 03/10] fftools/resources/resman: Remove unused function
parameter
Also constify in.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
fftools/resources/resman.c | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index f910535750..99d312bad8 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -65,7 +65,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in_len, char **out, size_t *out_len)
+static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
{
z_stream strm;
unsigned chunk = 65534;
@@ -108,8 +108,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, uint8_t *in, unsigned in
av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
}
- *out_len = chunk - strm.avail_out;
- buf[*out_len] = 0; // Ensure null termination
+ buf[chunk - strm.avail_out] = 0; // Ensure null termination
inflateEnd(&strm);
*out = (char *)buf;
@@ -154,9 +153,8 @@ char *ff_resman_get_string(FFResourceId resource_id)
#if CONFIG_RESOURCE_COMPRESSION
char *out = NULL;
- size_t out_len;
- int ret = decompress_zlib(ctx, (uint8_t *)resource_definition.data, *resource_definition.data_len, &out, &out_len);
+ int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
--
2.45.2
From 38436c69fc818b989594fb9912d4e99d8c4f3d1b Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Sun, 1 Jun 2025 23:19:29 +0200
Subject: [PATCH WIP 04/10] ffbuild/bin2c: Avoid reallocations when
uncompressing code,resources
This can be easily achieved by storing the uncompressed size
together with the actual zlib data. It also allows to use
zlib's uncompress() utility function.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
ffbuild/bin2c.c | 26 ++++++++++++---
fftools/resources/resman.c | 44 +++++--------------------
libavfilter/cuda/load_helper.c | 60 ++++++++--------------------------
3 files changed, 43 insertions(+), 87 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index a3e49f0a11..7bcb126c7e 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -38,7 +38,7 @@
#define MAX_BUF_SIZE UINT32_MAX
static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *compressed_sizep,
- FILE *input)
+ uint32_t *uncompressed_sizep, FILE *input)
{
z_stream zstream = { 0 };
int ret, zret = deflateInit(&zstream, 9);
@@ -46,7 +46,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
if (zret != Z_OK)
return -1;
- uint32_t buffer_size = 0, compressed_size = 0;
+ uint32_t buffer_size = 0, compressed_size = 0, uncompressed_size = 0;
unsigned char *compressed_data = NULL;
int flush = Z_NO_FLUSH;
@@ -56,6 +56,12 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
if (read < sizeof(tmp))
flush = Z_FINISH;
+ uncompressed_size += read;
+ if (uncompressed_size < read) { // overflow
+ ret = -1;
+ goto fail;
+ }
+
zstream.next_in = tmp;
zstream.avail_in = read;
@@ -92,6 +98,7 @@ static int read_file_and_compress(unsigned char **compressed_datap, uint32_t *co
deflateEnd(&zstream);
*compressed_datap = compressed_data;
*compressed_sizep = compressed_size;
+ *uncompressed_sizep = uncompressed_size;
return 0;
fail:
@@ -101,17 +108,28 @@ fail:
return ret;
}
+/// Write a 32bit integer according to the target's endianness.
+static void write_u32(FILE *output, uint32_t num)
+{
+ for (int i = 0; i < 4; ++i)
+ fprintf(output, "0x%02x, ",
+ (num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
+}
+
static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
{
unsigned char *compressed_data;
- uint32_t compressed_size;
+ uint32_t compressed_size, uncompressed_size;
- int err = read_file_and_compress(&compressed_data, &compressed_size, input);
+ int err = read_file_and_compress(&compressed_data, &compressed_size,
+ &uncompressed_size, input);
if (err)
return err;
*compressed_sizep = compressed_size;
+ write_u32(output, uncompressed_size);
+
for (unsigned i = 0; i < compressed_size; ++i)
fprintf(output, "0x%02x, ", compressed_data[i]);
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 99d312bad8..698529aca1 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -33,9 +33,8 @@
#include "resman.h"
#include "libavutil/avassert.h"
-#include "libavutil/pixdesc.h"
+#include "libavutil/intreadwrite.h"
#include "libavutil/dict.h"
-#include "libavutil/common.h"
extern const unsigned char ff_graph_html_data[];
extern const unsigned int ff_graph_html_len;
@@ -67,50 +66,23 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
{
- z_stream strm;
- unsigned chunk = 65534;
- int ret;
- uint8_t *buf;
-
- *out = NULL;
- memset(&strm, 0, sizeof(strm));
-
// Allocate output buffer with extra byte for null termination
- buf = (uint8_t *)av_mallocz(chunk + 1);
+ uint32_t uncompressed_size = AV_RN32(in);
+ uint8_t *buf = av_malloc(uncompressed_size + 1);
if (!buf) {
av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
return AVERROR(ENOMEM);
}
-
- // 15 + 16 tells zlib to detect GZIP or zlib automatically
- ret = inflateInit(&strm);
- if (ret != Z_OK) {
- av_log(ctx, AV_LOG_ERROR, "Error during zlib initialization: %s\n", strm.msg);
- av_free(buf);
- return AVERROR(ENOSYS);
- }
-
- strm.avail_in = in_len;
- strm.next_in = in;
- strm.avail_out = chunk;
- strm.next_out = buf;
-
- ret = inflate(&strm, Z_FINISH);
- if (ret != Z_OK && ret != Z_STREAM_END) {
- av_log(ctx, AV_LOG_ERROR, "Inflate failed: %d, %s\n", ret, strm.msg);
- inflateEnd(&strm);
+ uLongf buf_size = uncompressed_size;
+ int ret = uncompress(buf, &buf_size, in + 4, in_len);
+ if (ret != Z_OK || uncompressed_size != buf_size) {
+ av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
return AVERROR_EXTERNAL;
}
- if (strm.avail_out == 0) {
- // TODO: Error or loop decoding?
- av_log(ctx, AV_LOG_WARNING, "Decompression buffer may be too small\n");
- }
-
- buf[chunk - strm.avail_out] = 0; // Ensure null termination
+ buf[uncompressed_size] = 0; // Ensure null termination
- inflateEnd(&strm);
*out = (char *)buf;
return 0;
}
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index 7c62793077..0288ea49bb 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -21,11 +21,11 @@
#include "libavutil/hwcontext.h"
#include "libavutil/hwcontext_cuda_internal.h"
#include "libavutil/cuda_check.h"
+#include "libavutil/intreadwrite.h"
#include "libavutil/mem.h"
#if CONFIG_PTX_COMPRESSION
#include <zlib.h>
-#define CHUNK_SIZE 1024 * 64
#endif
#include "load_helper.h"
@@ -38,57 +38,23 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
CudaFunctions *cu = hwctx->internal->cuda_dl;
#if CONFIG_PTX_COMPRESSION
- z_stream stream = { 0 };
- uint8_t *buf, *tmp;
- uint64_t buf_size;
- int ret;
-
- if (inflateInit(&stream) != Z_OK) {
- av_log(avctx, AV_LOG_ERROR, "Error during zlib initialisation: %s\n", stream.msg);
- return AVERROR(ENOSYS);
- }
-
- buf_size = CHUNK_SIZE * 4;
- buf = av_realloc(NULL, buf_size);
- if (!buf) {
- inflateEnd(&stream);
+ uint32_t uncompressed_size = AV_RN32(data);
+ uint8_t *buf = av_realloc(NULL, uncompressed_size + 1);
+ if (!buf)
return AVERROR(ENOMEM);
- }
-
- stream.next_in = data;
- stream.avail_in = length;
-
- do {
- stream.avail_out = buf_size - stream.total_out;
- stream.next_out = buf + stream.total_out;
-
- ret = inflate(&stream, Z_FINISH);
- if (ret != Z_OK && ret != Z_STREAM_END && ret != Z_BUF_ERROR) {
- av_log(avctx, AV_LOG_ERROR, "zlib inflate error(%d): %s\n", ret, stream.msg);
- inflateEnd(&stream);
- av_free(buf);
- return AVERROR(EINVAL);
- }
-
- if (stream.avail_out == 0) {
- buf_size += CHUNK_SIZE;
- tmp = av_realloc(buf, buf_size);
- if (!tmp) {
- inflateEnd(&stream);
- av_free(buf);
- return AVERROR(ENOMEM);
- }
- buf = tmp;
- }
- } while (ret != Z_STREAM_END);
+ uLongf buf_size = uncompressed_size;
+ int ret = uncompress(buf, &buf_size, data + 4, length);
+ if (ret != Z_OK || uncompressed_size != buf_size) {
+ av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
+ ret = AVERROR_EXTERNAL;
+ goto fail;
+ }
// NULL-terminate string
- // there is guaranteed to be space for this, due to condition in loop
- buf[stream.total_out] = 0;
-
- inflateEnd(&stream);
+ buf[uncompressed_size] = 0;
ret = CHECK_CU(cu->cuModuleLoadData(cu_module, buf));
+fail:
av_free(buf);
return ret;
#else
--
2.45.2
From 89c5a836ebac6b9073fc377351d57ebd141c3271 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Sun, 1 Jun 2025 23:37:54 +0200
Subject: [PATCH WIP 05/10] ffbuild/bin2c: Store compressed size in-band
This is simpler than using a separate constant.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
ffbuild/bin2c.c | 10 +++-------
fftools/resources/resman.c | 14 +++++---------
fftools/resources/resman.h | 2 --
libavfilter/cuda/load_helper.c | 4 ++--
libavfilter/cuda/load_helper.h | 2 +-
libavfilter/vf_bilateral_cuda.c | 3 +--
libavfilter/vf_bwdif_cuda.c | 3 +--
libavfilter/vf_chromakey_cuda.c | 3 +--
libavfilter/vf_colorspace_cuda.c | 4 +---
libavfilter/vf_overlay_cuda.c | 3 +--
libavfilter/vf_scale_cuda.c | 3 +--
libavfilter/vf_thumbnail_cuda.c | 3 +--
libavfilter/vf_yadif_cuda.c | 3 +--
13 files changed, 19 insertions(+), 38 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index 7bcb126c7e..fcff9d4ab7 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -116,7 +116,7 @@ static void write_u32(FILE *output, uint32_t num)
(num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
}
-static int handle_compressed_file(FILE *input, FILE *output, unsigned *compressed_sizep)
+static int handle_compressed_file(FILE *input, FILE *output)
{
unsigned char *compressed_data;
uint32_t compressed_size, uncompressed_size;
@@ -126,9 +126,8 @@ static int handle_compressed_file(FILE *input, FILE *output, unsigned *compresse
if (err)
return err;
- *compressed_sizep = compressed_size;
-
write_u32(output, uncompressed_size);
+ write_u32(output, compressed_size);
for (unsigned i = 0; i < compressed_size; ++i)
fprintf(output, "0x%02x, ", compressed_data[i]);
@@ -143,7 +142,6 @@ int main(int argc, char **argv)
{
const char *name;
FILE *input, *output;
- unsigned int length = 0;
unsigned char data;
av_unused int compression = 0;
int arg_idx = 1;
@@ -193,7 +191,7 @@ int main(int argc, char **argv)
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
if (compression) {
- int err = handle_compressed_file(input, output, &length);
+ int err = handle_compressed_file(input, output);
if (err) {
fclose(input);
fclose(output);
@@ -204,12 +202,10 @@ int main(int argc, char **argv)
{
while (fread(&data, 1, 1, input) > 0) {
fprintf(output, "0x%02x, ", data);
- length++;
}
}
fprintf(output, "0x00 };\n");
- fprintf(output, "const unsigned int ff_%s_len = %u;\n", name, length);
fclose(output);
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 698529aca1..fc00465775 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -37,14 +37,11 @@
#include "libavutil/dict.h"
extern const unsigned char ff_graph_html_data[];
-extern const unsigned int ff_graph_html_len;
-
extern const unsigned char ff_graph_css_data[];
-extern const unsigned ff_graph_css_len;
static const FFResourceDefinition resource_definitions[] = {
- [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", &ff_graph_css_data[0], &ff_graph_css_len },
- [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", &ff_graph_html_data[0], &ff_graph_html_len },
+ [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data },
+ [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data },
};
@@ -64,7 +61,7 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
#if CONFIG_RESOURCE_COMPRESSION
-static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsigned in_len, char **out)
+static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
uint32_t uncompressed_size = AV_RN32(in);
@@ -74,7 +71,7 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, unsig
return AVERROR(ENOMEM);
}
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, in + 4, in_len);
+ int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
@@ -126,8 +123,7 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
- int ret = decompress_zlib(ctx, resource_definition.data, *resource_definition.data_len, &out);
-
+ int ret = decompress_zlib(ctx, resource_definition.data, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index 6485db5091..ad57a10204 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -39,8 +39,6 @@ typedef struct FFResourceDefinition {
const char *name;
const unsigned char *data;
- const unsigned *data_len;
-
} FFResourceDefinition;
void ff_resman_uninit(void);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index 0288ea49bb..e47f0421ef 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -33,7 +33,7 @@
#define CHECK_CU(x) FF_CUDA_CHECK_DL(avctx, cu, x)
int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module,
- const unsigned char *data, const unsigned int length)
+ const unsigned char *data)
{
CudaFunctions *cu = hwctx->internal->cuda_dl;
@@ -44,7 +44,7 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
return AVERROR(ENOMEM);
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, data + 4, length);
+ int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
ret = AVERROR_EXTERNAL;
diff --git a/libavfilter/cuda/load_helper.h b/libavfilter/cuda/load_helper.h
index 455bf36a23..d25aa0ae27 100644
--- a/libavfilter/cuda/load_helper.h
+++ b/libavfilter/cuda/load_helper.h
@@ -23,6 +23,6 @@
* Loads a CUDA module and applies any decompression, if necessary.
*/
int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_module,
- const unsigned char *data, const unsigned int length);
+ const unsigned char *data);
#endif /* AVFILTER_CUDA_LOAD_HELPER_H */
diff --git a/libavfilter/vf_bilateral_cuda.c b/libavfilter/vf_bilateral_cuda.c
index 7115fa9e05..50ddaad213 100644
--- a/libavfilter/vf_bilateral_cuda.c
+++ b/libavfilter/vf_bilateral_cuda.c
@@ -217,14 +217,13 @@ static av_cold int cuda_bilateral_load_functions(AVFilterContext *ctx)
int ret;
extern const unsigned char ff_vf_bilateral_cuda_ptx_data[];
- extern const unsigned int ff_vf_bilateral_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_bilateral_cuda_ptx_data, ff_vf_bilateral_cuda_ptx_len);
+ ff_vf_bilateral_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c
index cc954ab1d3..82c46886f3 100644
--- a/libavfilter/vf_bwdif_cuda.c
+++ b/libavfilter/vf_bwdif_cuda.c
@@ -29,7 +29,6 @@
#include "cuda/load_helper.h"
extern const unsigned char ff_vf_bwdif_cuda_ptx_data[];
-extern const unsigned int ff_vf_bwdif_cuda_ptx_len;
typedef struct DeintCUDAContext {
YADIFContext yadif;
@@ -305,7 +304,7 @@ static int config_output(AVFilterLink *link)
if (ret < 0)
goto exit;
- ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data, ff_vf_bwdif_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_bwdif_cuda_ptx_data);
if (ret < 0)
goto exit;
diff --git a/libavfilter/vf_chromakey_cuda.c b/libavfilter/vf_chromakey_cuda.c
index 43f50c5a9a..e606da01ac 100644
--- a/libavfilter/vf_chromakey_cuda.c
+++ b/libavfilter/vf_chromakey_cuda.c
@@ -220,14 +220,13 @@ static av_cold int cudachromakey_load_functions(AVFilterContext *ctx)
int ret;
extern const unsigned char ff_vf_chromakey_cuda_ptx_data[];
- extern const unsigned int ff_vf_chromakey_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_chromakey_cuda_ptx_data, ff_vf_chromakey_cuda_ptx_len);
+ ff_vf_chromakey_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_colorspace_cuda.c b/libavfilter/vf_colorspace_cuda.c
index 54d6228cd1..2d922430e4 100644
--- a/libavfilter/vf_colorspace_cuda.c
+++ b/libavfilter/vf_colorspace_cuda.c
@@ -198,15 +198,13 @@ static av_cold int cudacolorspace_load_functions(AVFilterContext* ctx)
int ret;
extern const unsigned char ff_vf_colorspace_cuda_ptx_data[];
- extern const unsigned int ff_vf_colorspace_cuda_ptx_len;
ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx));
if (ret < 0)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_colorspace_cuda_ptx_data,
- ff_vf_colorspace_cuda_ptx_len);
+ ff_vf_colorspace_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_overlay_cuda.c b/libavfilter/vf_overlay_cuda.c
index 4708b34d30..77da5b5e01 100644
--- a/libavfilter/vf_overlay_cuda.c
+++ b/libavfilter/vf_overlay_cuda.c
@@ -415,7 +415,6 @@ static int overlay_cuda_activate(AVFilterContext *avctx)
static int overlay_cuda_config_output(AVFilterLink *outlink)
{
extern const unsigned char ff_vf_overlay_cuda_ptx_data[];
- extern const unsigned int ff_vf_overlay_cuda_ptx_len;
int err;
FilterLink *outl = ff_filter_link(outlink);
@@ -494,7 +493,7 @@ static int overlay_cuda_config_output(AVFilterLink *outlink)
return err;
}
- err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data, ff_vf_overlay_cuda_ptx_len);
+ err = ff_cuda_load_module(ctx, ctx->hwctx, &ctx->cu_module, ff_vf_overlay_cuda_ptx_data);
if (err < 0) {
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
return err;
diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c
index 44eef207ca..6e238c7c73 100644
--- a/libavfilter/vf_scale_cuda.c
+++ b/libavfilter/vf_scale_cuda.c
@@ -290,7 +290,6 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
const char *function_infix = "";
extern const unsigned char ff_vf_scale_cuda_ptx_data[];
- extern const unsigned int ff_vf_scale_cuda_ptx_len;
switch(s->interp_algo) {
case INTERP_ALGO_NEAREST:
@@ -324,7 +323,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx)
return ret;
ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module,
- ff_vf_scale_cuda_ptx_data, ff_vf_scale_cuda_ptx_len);
+ ff_vf_scale_cuda_ptx_data);
if (ret < 0)
goto fail;
diff --git a/libavfilter/vf_thumbnail_cuda.c b/libavfilter/vf_thumbnail_cuda.c
index 121274de11..bea07f7041 100644
--- a/libavfilter/vf_thumbnail_cuda.c
+++ b/libavfilter/vf_thumbnail_cuda.c
@@ -368,7 +368,6 @@ static int config_props(AVFilterLink *inlink)
int ret;
extern const unsigned char ff_vf_thumbnail_cuda_ptx_data[];
- extern const unsigned int ff_vf_thumbnail_cuda_ptx_len;
s->hwctx = device_hwctx;
s->cu_stream = s->hwctx->stream;
@@ -377,7 +376,7 @@ static int config_props(AVFilterLink *inlink)
if (ret < 0)
return ret;
- ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data, ff_vf_thumbnail_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, device_hwctx, &s->cu_module, ff_vf_thumbnail_cuda_ptx_data);
if (ret < 0)
return ret;
diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c
index 50ac61ad8a..cd9a5ba2e7 100644
--- a/libavfilter/vf_yadif_cuda.c
+++ b/libavfilter/vf_yadif_cuda.c
@@ -29,7 +29,6 @@
#include "cuda/load_helper.h"
extern const unsigned char ff_vf_yadif_cuda_ptx_data[];
-extern const unsigned int ff_vf_yadif_cuda_ptx_len;
typedef struct DeintCUDAContext {
YADIFContext yadif;
@@ -291,7 +290,7 @@ static int config_output(AVFilterLink *link)
if (ret < 0)
goto exit;
- ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data, ff_vf_yadif_cuda_ptx_len);
+ ret = ff_cuda_load_module(ctx, s->hwctx, &s->cu_module, ff_vf_yadif_cuda_ptx_data);
if (ret < 0)
goto exit;
--
2.45.2
From ced36923900a289ca0c74bdfc9a34b35154f4003 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 00:08:31 +0200
Subject: [PATCH WIP 06/10] avfilter/cuda/load_helper: Use aligned reads
Possible if one aligns the buffer.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
ffbuild/bin2c.c | 10 ++++++----
fftools/resources/resman.c | 4 ++--
libavfilter/cuda/load_helper.c | 4 ++--
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/ffbuild/bin2c.c b/ffbuild/bin2c.c
index fcff9d4ab7..1f1bce3a97 100644
--- a/ffbuild/bin2c.c
+++ b/ffbuild/bin2c.c
@@ -116,11 +116,14 @@ static void write_u32(FILE *output, uint32_t num)
(num >> (HAVE_BIGENDIAN ? (24 - 8 * i) : 8 * i)) & 0xff);
}
-static int handle_compressed_file(FILE *input, FILE *output)
+static int handle_compressed_file(FILE *input, FILE *output, const char *name)
{
unsigned char *compressed_data;
uint32_t compressed_size, uncompressed_size;
+ fprintf(output, "#include \"libavutil/mem_internal.h\"\n");
+ fprintf(output, "DECLARE_ALIGNED_4(const unsigned char, ff_%s_data)[] = { ", name);
+
int err = read_file_and_compress(&compressed_data, &compressed_size,
&uncompressed_size, input);
if (err)
@@ -187,11 +190,9 @@ int main(int argc, char **argv)
}
}
- fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
-
#if CONFIG_PTX_COMPRESSION || CONFIG_RESOURCE_COMPRESSION
if (compression) {
- int err = handle_compressed_file(input, output);
+ int err = handle_compressed_file(input, output, name);
if (err) {
fclose(input);
fclose(output);
@@ -200,6 +201,7 @@ int main(int argc, char **argv)
} else
#endif
{
+ fprintf(output, "const unsigned char ff_%s_data[] = { ", name);
while (fread(&data, 1, 1, input) > 0) {
fprintf(output, "0x%02x, ", data);
}
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index fc00465775..0094ecd4c5 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -64,14 +64,14 @@ static ResourceManagerContext resman_ctx = { .class = &resman_class };
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
- uint32_t uncompressed_size = AV_RN32(in);
+ uint32_t uncompressed_size = AV_RN32A(in);
uint8_t *buf = av_malloc(uncompressed_size + 1);
if (!buf) {
av_log(ctx, AV_LOG_ERROR, "Failed to allocate decompression buffer\n");
return AVERROR(ENOMEM);
}
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, in + 8, AV_RN32(in + 4));
+ int ret = uncompress(buf, &buf_size, in + 8, AV_RN32A(in + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(ctx, AV_LOG_ERROR, "Error uncompressing resource. zlib returned %d\n", ret);
av_free(buf);
diff --git a/libavfilter/cuda/load_helper.c b/libavfilter/cuda/load_helper.c
index e47f0421ef..79ec37c7cd 100644
--- a/libavfilter/cuda/load_helper.c
+++ b/libavfilter/cuda/load_helper.c
@@ -38,13 +38,13 @@ int ff_cuda_load_module(void *avctx, AVCUDADeviceContext *hwctx, CUmodule *cu_mo
CudaFunctions *cu = hwctx->internal->cuda_dl;
#if CONFIG_PTX_COMPRESSION
- uint32_t uncompressed_size = AV_RN32(data);
+ uint32_t uncompressed_size = AV_RN32A(data);
uint8_t *buf = av_realloc(NULL, uncompressed_size + 1);
if (!buf)
return AVERROR(ENOMEM);
uLongf buf_size = uncompressed_size;
- int ret = uncompress(buf, &buf_size, data + 8, AV_RN32(data + 4));
+ int ret = uncompress(buf, &buf_size, data + 8, AV_RN32A(data + 4));
if (ret != Z_OK || uncompressed_size != buf_size) {
av_log(avctx, AV_LOG_ERROR, "Error uncompressing cuda code. zlib returned %d\n", ret);
ret = AVERROR_EXTERNAL;
--
2.45.2
From dbb59f19565db0cb9c69296487bd7432d53b634b Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 03:09:45 +0200
Subject: [PATCH WIP 07/10] fftools/resources/resman: Use pointer to resource
definition
It does not really matter on its own given that FFResourceDefinition
is so small; it is mostly in preparation for the next commit.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
fftools/resources/resman.c | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 0094ecd4c5..1de98475c3 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -98,23 +98,23 @@ void ff_resman_uninit(void)
char *ff_resman_get_string(FFResourceId resource_id)
{
ResourceManagerContext *ctx = &resman_ctx;
- FFResourceDefinition resource_definition = { 0 };
+ const FFResourceDefinition *resource_definition = NULL;
AVDictionaryEntry *dic_entry;
char *res = NULL;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
- FFResourceDefinition def = resource_definitions[i];
- if (def.resource_id == resource_id) {
+ const FFResourceDefinition *def = &resource_definitions[i];
+ if (def->resource_id == resource_id) {
resource_definition = def;
break;
}
}
- av_assert1(resource_definition.name);
+ av_assert1(resource_definition);
ff_mutex_lock(&mutex);
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
+ dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
if (!dic_entry) {
int dict_ret;
@@ -123,13 +123,13 @@ char *ff_resman_get_string(FFResourceId resource_id)
char *out = NULL;
- int ret = decompress_zlib(ctx, resource_definition.data, &out);
+ int ret = decompress_zlib(ctx, resource_definition->data, &out);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
}
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, out, 0);
+ dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, out, 0);
if (dict_ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
av_freep(&out);
@@ -139,14 +139,14 @@ char *ff_resman_get_string(FFResourceId resource_id)
av_freep(&out);
#else
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition.name, (const char *)resource_definition.data, 0);
+ dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, (const char *)resource_definition->data, 0);
if (dict_ret < 0) {
av_log(ctx, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
goto end;
}
#endif
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition.name, NULL, 0);
+ dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
if (!dic_entry) {
av_log(ctx, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
--
2.45.2
From 19f4c569b687e4568513630f15f281774ce650e4 Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 03:14:36 +0200
Subject: [PATCH WIP 08/10] fftools/resources/resman: Constify ret of
ff_resman_get_string()
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
fftools/resources/resman.c | 4 ++--
fftools/resources/resman.h | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 1de98475c3..aa99e07f50 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -95,12 +95,12 @@ void ff_resman_uninit(void)
}
-char *ff_resman_get_string(FFResourceId resource_id)
+const char *ff_resman_get_string(FFResourceId resource_id)
{
ResourceManagerContext *ctx = &resman_ctx;
const FFResourceDefinition *resource_definition = NULL;
AVDictionaryEntry *dic_entry;
- char *res = NULL;
+ const char *res = NULL;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
const FFResourceDefinition *def = &resource_definitions[i];
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index ad57a10204..d41c89ce37 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -43,6 +43,6 @@ typedef struct FFResourceDefinition {
void ff_resman_uninit(void);
-char *ff_resman_get_string(FFResourceId resource_id);
+const char *ff_resman_get_string(FFResourceId resource_id);
#endif /* FFTOOLS_RESOURCES_RESMAN_H */
--
2.45.2
From 66a22cb4bd701183cf06272d315a70ab6d6221df Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 03:37:43 +0200
Subject: [PATCH WIP 09/10] fftools/resources/resman: Avoid AVDictionary
Using it incurs an unnecessary duplication (temporary when
compression is enabled). It furthermore incurs checks
for av_dict_set()/av_dict_get(). Using an array of pointers
is simpler.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
fftools/resources/resman.c | 64 +++++++++++---------------------------
fftools/resources/resman.h | 1 -
2 files changed, 18 insertions(+), 47 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index aa99e07f50..4497d65f3d 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -34,24 +34,23 @@
#include "resman.h"
#include "libavutil/avassert.h"
#include "libavutil/intreadwrite.h"
-#include "libavutil/dict.h"
extern const unsigned char ff_graph_html_data[];
extern const unsigned char ff_graph_css_data[];
static const FFResourceDefinition resource_definitions[] = {
- [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, "graph.css", ff_graph_css_data },
- [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, "graph.html", ff_graph_html_data },
+ [FF_RESOURCE_GRAPH_CSS] = { FF_RESOURCE_GRAPH_CSS, ff_graph_css_data },
+ [FF_RESOURCE_GRAPH_HTML] = { FF_RESOURCE_GRAPH_HTML, ff_graph_html_data },
};
-
+#if CONFIG_RESOURCE_COMPRESSION
static const AVClass resman_class = {
.class_name = "ResourceManager",
};
typedef struct ResourceManagerContext {
const AVClass *class;
- AVDictionary *resource_dic;
+ char *resources[FF_ARRAY_ELEMS(resource_definitions)];
} ResourceManagerContext;
static AVMutex mutex = AV_MUTEX_INITIALIZER;
@@ -59,8 +58,6 @@ static AVMutex mutex = AV_MUTEX_INITIALIZER;
static ResourceManagerContext resman_ctx = { .class = &resman_class };
-#if CONFIG_RESOURCE_COMPRESSION
-
static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char **out)
{
// Allocate output buffer with extra byte for null termination
@@ -87,76 +84,51 @@ static int decompress_zlib(ResourceManagerContext *ctx, const uint8_t *in, char
void ff_resman_uninit(void)
{
+#if CONFIG_RESOURCE_COMPRESSION
ff_mutex_lock(&mutex);
- av_dict_free(&resman_ctx.resource_dic);
+ for (size_t i = 0; i < FF_ARRAY_ELEMS(resman_ctx.resources); ++i)
+ av_freep(&resman_ctx.resources[i]);
ff_mutex_unlock(&mutex);
+#endif
}
const char *ff_resman_get_string(FFResourceId resource_id)
{
- ResourceManagerContext *ctx = &resman_ctx;
const FFResourceDefinition *resource_definition = NULL;
- AVDictionaryEntry *dic_entry;
const char *res = NULL;
+ av_unused unsigned idx;
for (unsigned i = 0; i < FF_ARRAY_ELEMS(resource_definitions); ++i) {
const FFResourceDefinition *def = &resource_definitions[i];
if (def->resource_id == resource_id) {
resource_definition = def;
+ idx = i;
break;
}
}
av_assert1(resource_definition);
- ff_mutex_lock(&mutex);
-
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
-
- if (!dic_entry) {
- int dict_ret;
-
#if CONFIG_RESOURCE_COMPRESSION
+ ff_mutex_lock(&mutex);
- char *out = NULL;
+ ResourceManagerContext *ctx = &resman_ctx;
- int ret = decompress_zlib(ctx, resource_definition->data, &out);
+ if (!ctx->resources[idx]) {
+ int ret = decompress_zlib(ctx, resource_definition->data, &ctx->resources[idx]);
if (ret) {
av_log(ctx, AV_LOG_ERROR, "Unable to decompress the resource with ID %d\n", resource_id);
goto end;
}
-
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, out, 0);
- if (dict_ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to store decompressed resource in dictionary: %d\n", dict_ret);
- av_freep(&out);
- goto end;
- }
-
- av_freep(&out);
-#else
-
- dict_ret = av_dict_set(&ctx->resource_dic, resource_definition->name, (const char *)resource_definition->data, 0);
- if (dict_ret < 0) {
- av_log(ctx, AV_LOG_ERROR, "Failed to store resource in dictionary: %d\n", dict_ret);
- goto end;
- }
-
-#endif
- dic_entry = av_dict_get(ctx->resource_dic, resource_definition->name, NULL, 0);
-
- if (!dic_entry) {
- av_log(ctx, AV_LOG_ERROR, "Failed to retrieve resource from dictionary after storing it\n");
- goto end;
- }
}
-
- res = dic_entry->value;
-
+ res = ctx->resources[idx];
end:
ff_mutex_unlock(&mutex);
+#else
+ res = resource_definition->data;
+#endif
return res;
}
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index d41c89ce37..ee22e960d3 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -36,7 +36,6 @@ typedef enum {
typedef struct FFResourceDefinition {
FFResourceId resource_id;
- const char *name;
const unsigned char *data;
} FFResourceDefinition;
--
2.45.2
From c4e1a52e6170f1d88bfef26fb4cfdaff470c961a Mon Sep 17 00:00:00 2001
From: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
Date: Mon, 2 Jun 2025 03:48:50 +0200
Subject: [PATCH WIP 10/10] fftools/resources/resman: Improve headers
Don't rely on implicit inclusions of lavu/thread.h and lavu/mem.h.
Signed-off-by: Andreas Rheinhardt <andreas.rheinha...@outlook.com>
---
fftools/resources/resman.c | 5 ++++-
fftools/resources/resman.h | 8 --------
2 files changed, 4 insertions(+), 9 deletions(-)
diff --git a/fftools/resources/resman.c b/fftools/resources/resman.c
index 4497d65f3d..9b8f6e406a 100644
--- a/fftools/resources/resman.c
+++ b/fftools/resources/resman.c
@@ -25,10 +25,13 @@
#include "config.h"
-#include <string.h>
+#include <stddef.h>
#if CONFIG_RESOURCE_COMPRESSION
#include <zlib.h>
+#include "libavutil/intreadwrite.h"
+#include "libavutil/mem.h"
+#include "libavutil/thread.h"
#endif
#include "resman.h"
diff --git a/fftools/resources/resman.h b/fftools/resources/resman.h
index ee22e960d3..20e6b1037c 100644
--- a/fftools/resources/resman.h
+++ b/fftools/resources/resman.h
@@ -21,14 +21,6 @@
#ifndef FFTOOLS_RESOURCES_RESMAN_H
#define FFTOOLS_RESOURCES_RESMAN_H
-#include <stdint.h>
-
-#include "config.h"
-#include "fftools/ffmpeg.h"
-#include "libavutil/avutil.h"
-#include "libavutil/bprint.h"
-#include "fftools/textformat/avtextformat.h"
-
typedef enum {
FF_RESOURCE_GRAPH_CSS,
FF_RESOURCE_GRAPH_HTML,
--
2.45.2
_______________________________________________
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
To unsubscribe, visit link above, or email
ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".
