On Wed, Mar 23, 2022 at 12:24 PM Mark Gaiser <mark...@gmail.com> wrote:
> On Fri, Mar 18, 2022 at 3:59 PM Mark Gaiser <mark...@gmail.com> wrote: > >> On Fri, Mar 18, 2022 at 3:50 PM Mark Gaiser <mark...@gmail.com> wrote: >> >>> This patch adds support for: >>> - ffplay ipfs://<cid> >>> - ffplay ipns://<cid> >>> >>> IPFS data can be played from so called "ipfs gateways". >>> A gateway is essentially a webserver that gives access to the >>> distributed IPFS network. >>> >>> This protocol support (ipfs and ipns) therefore translates >>> ipfs:// and ipns:// to a http:// url. This resulting url is >>> then handled by the http protocol. It could also be https >>> depending on the gateway provided. >>> >>> To use this protocol, a gateway must be provided. >>> If you do nothing it will try to find it in your >>> $HOME/.ipfs/gateway file. The ways to set it manually are: >>> 1. Define a -gateway <url> to the gateway. >>> 2. Define $IPFS_GATEWAY with the full http link to the gateway. >>> 3. Define $IPFS_PATH and point it to the IPFS data path. >>> 4. Have IPFS running in your local user folder (under $HOME/.ipfs). >>> >>> Signed-off-by: Mark Gaiser <mark...@gmail.com> >>> --- >>> configure | 2 + >>> doc/protocols.texi | 30 ++++ >>> libavformat/Makefile | 2 + >>> libavformat/ipfsgateway.c | 310 ++++++++++++++++++++++++++++++++++++++ >>> libavformat/protocols.c | 2 + >>> 5 files changed, 346 insertions(+) >>> create mode 100644 libavformat/ipfsgateway.c >>> >>> diff --git a/configure b/configure >>> index 5b19a35f59..6ff09e7974 100755 >>> --- a/configure >>> +++ b/configure >>> @@ -3585,6 +3585,8 @@ udp_protocol_select="network" >>> udplite_protocol_select="network" >>> unix_protocol_deps="sys_un_h" >>> unix_protocol_select="network" >>> +ipfs_protocol_select="https_protocol" >>> +ipns_protocol_select="https_protocol" >>> >>> # external library protocols >>> libamqp_protocol_deps="librabbitmq" >>> diff --git a/doc/protocols.texi b/doc/protocols.texi >>> index d207df0b52..7c9c0a4808 100644 >>> --- a/doc/protocols.texi >>> +++ b/doc/protocols.texi >>> @@ -2025,5 +2025,35 @@ decoding errors. >>> >>> @end table >>> >>> +@section ipfs >>> + >>> +InterPlanetary File System (IPFS) protocol support. One can access >>> files stored >>> +on the IPFS network through so called gateways. Those are http(s) >>> endpoints. >>> +This protocol wraps the IPFS native protocols (ipfs:// and ipns://) to >>> be send >>> +to such a gateway. Users can (and should) host their own node which >>> means this >>> +protocol will use your local machine gateway to access files on the >>> IPFS network. >>> + >>> +If a user doesn't have a node of their own then the public gateway >>> dweb.link is >>> +used by default. >>> + >>> +You can use this protocol in 2 ways. Using IPFS: >>> +@example >>> +ffplay ipfs://QmbGtJg23skhvFmu9mJiePVByhfzu5rwo74MEkVDYAmF5T >>> +@end example >>> + >>> +Or the IPNS protocol (IPNS is mutable IPFS): >>> +@example >>> +ffplay ipns://QmbGtJg23skhvFmu9mJiePVByhfzu5rwo74MEkVDYAmF5T >>> +@end example >>> + >>> +You can also change the gateway to be used: >>> + >>> +@table @option >>> + >>> +@item gateway >>> +Defines the gateway to use. When nothing is provided the protocol will >>> first try >>> +your local gateway. If that fails dweb.link will be used. >>> + >>> +@end table >>> >>> @c man end PROTOCOLS >>> diff --git a/libavformat/Makefile b/libavformat/Makefile >>> index 3dc6a479cc..4edce8420f 100644 >>> --- a/libavformat/Makefile >>> +++ b/libavformat/Makefile >>> @@ -656,6 +656,8 @@ OBJS-$(CONFIG_SRTP_PROTOCOL) += >>> srtpproto.o srtp.o >>> OBJS-$(CONFIG_SUBFILE_PROTOCOL) += subfile.o >>> OBJS-$(CONFIG_TEE_PROTOCOL) += teeproto.o tee_common.o >>> OBJS-$(CONFIG_TCP_PROTOCOL) += tcp.o >>> +OBJS-$(CONFIG_IPFS_PROTOCOL) += ipfsgateway.o >>> +OBJS-$(CONFIG_IPNS_PROTOCOL) += ipfsgateway.o >>> TLS-OBJS-$(CONFIG_GNUTLS) += tls_gnutls.o >>> TLS-OBJS-$(CONFIG_LIBTLS) += tls_libtls.o >>> TLS-OBJS-$(CONFIG_MBEDTLS) += tls_mbedtls.o >>> diff --git a/libavformat/ipfsgateway.c b/libavformat/ipfsgateway.c >>> new file mode 100644 >>> index 0000000000..2b2905d2c2 >>> --- /dev/null >>> +++ b/libavformat/ipfsgateway.c >>> @@ -0,0 +1,310 @@ >>> +/* >>> + * IPFS and IPNS protocol support through IPFS Gateway. >>> + * Copyright (c) 2022 Mark Gaiser >>> + * >>> + * This file is part of FFmpeg. >>> + * >>> + * FFmpeg is free software; you can redistribute it and/or >>> + * modify it under the terms of the GNU Lesser General Public >>> + * License as published by the Free Software Foundation; either >>> + * version 2.1 of the License, or (at your option) any later version. >>> + * >>> + * FFmpeg is distributed in the hope that it will be useful, >>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU >>> + * Lesser General Public License for more details. >>> + * >>> + * You should have received a copy of the GNU Lesser General Public >>> + * License along with FFmpeg; if not, write to the Free Software >>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA >>> 02110-1301 USA >>> + */ >>> + >>> +#include "avformat.h" >>> +#include "libavutil/avassert.h" >>> +#include "libavutil/avstring.h" >>> +#include "libavutil/internal.h" >>> +#include "libavutil/opt.h" >>> +#include "libavutil/tree.h" >>> +#include <fcntl.h> >>> +#if HAVE_IO_H >>> +#include <io.h> >>> +#endif >>> +#if HAVE_UNISTD_H >>> +#include <unistd.h> >>> +#endif >>> +#include "os_support.h" >>> +#include "url.h" >>> +#include <stdlib.h> >>> +#include <sys/stat.h> >>> + >>> +typedef struct IPFSGatewayContext { >>> + AVClass *class; >>> + URLContext *inner; >>> + // Is filled by the -gateway argument and not changed after. >>> + char *gateway; >>> + // If the above gateway is non null, it will be copied into this >>> buffer. >>> + // Else this buffer will contain the auto detected gateway. >>> + // In either case, the gateway to use will be in this buffer. >>> + char gateway_buffer[PATH_MAX]; >>> +} IPFSGatewayContext; >>> + >>> +// A best-effort way to find the IPFS gateway. >>> +// Only the most appropiate gateway is set. It's not actually requested >>> +// (http call) to prevent a potential slowdown in startup. A potential >>> timeout >>> +// is handled by the HTTP protocol. >>> +static int populate_ipfs_gateway(URLContext *h) >>> +{ >>> + IPFSGatewayContext *c = h->priv_data; >>> + char ipfs_full_data_folder[PATH_MAX]; >>> + char ipfs_gateway_file[PATH_MAX]; >>> + struct stat st; >>> + int stat_ret = 0; >>> + int ret = AVERROR(EINVAL); >>> + FILE *gateway_file = NULL; >>> + >>> + // Test $IPFS_GATEWAY. >>> + if (getenv("IPFS_GATEWAY") != NULL) { >>> + if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s", >>> + getenv("IPFS_GATEWAY")) >= >>> sizeof(c->gateway_buffer)) { >>> + av_log(h, AV_LOG_WARNING, "The IPFS_GATEWAY environment >>> variable exceeds the maximum length. We allow a max of %zu characters\n", >>> sizeof(c->gateway_buffer)); >>> + ret = AVERROR(EINVAL); >>> + goto err; >>> + } >>> + >>> + ret = 1; >>> + goto err; >>> + } else >>> + av_log(h, AV_LOG_DEBUG, "$IPFS_GATEWAY is empty.\n"); >>> + >>> + // We need to know the IPFS folder to - eventually - read the >>> contents of >>> + // the "gateway" file which would tell us the gateway to use. >>> + if (getenv("IPFS_PATH") == NULL) { >>> + av_log(h, AV_LOG_DEBUG, "$IPFS_PATH is empty.\n"); >>> + >>> + // Try via the home folder. >>> + if (getenv("HOME") == NULL) { >>> + av_log(h, AV_LOG_WARNING, "$HOME appears to be empty.\n"); >>> + ret = AVERROR(EINVAL); >>> + goto err; >>> + } >>> + >>> + // Verify the composed path fits. >>> + if (snprintf(ipfs_full_data_folder, >>> sizeof(ipfs_full_data_folder), >>> + "%s/.ipfs/", getenv("HOME")) >= >>> sizeof(ipfs_full_data_folder)) { >>> + av_log(h, AV_LOG_WARNING, "The IPFS data path exceeds the >>> max path length (%zu)\n", sizeof(ipfs_full_data_folder)); >>> + ret = AVERROR(EINVAL); >>> + goto err; >>> + } >>> + >>> + // Stat the folder. >>> + // It should exist in a default IPFS setup when run as local >>> user. >>> +#ifndef _WIN32 >>> + stat_ret = stat(ipfs_full_data_folder, &st); >>> +#else >>> + stat_ret = win32_stat(ipfs_full_data_folder, &st); >>> +#endif >>> + if (stat_ret < 0) { >>> + av_log(h, AV_LOG_INFO, "Unable to find IPFS folder. We >>> tried:\n"); >>> + av_log(h, AV_LOG_INFO, "- $IPFS_PATH, which was empty.\n"); >>> + av_log(h, AV_LOG_INFO, "- $HOME/.ipfs (full uri: %s) which >>> doesn't exist.\n", ipfs_full_data_folder); >>> + ret = AVERROR(ENOENT); >>> + goto err; >>> + } >>> + } else { >>> + if (snprintf(ipfs_full_data_folder, >>> sizeof(ipfs_full_data_folder), "%s", >>> + getenv("IPFS_PATH")) >= sizeof(ipfs_full_data_folder)) >>> { >>> + av_log(h, AV_LOG_WARNING, "The IPFS_PATH environment >>> variable exceeds the maximum length. We allow a max of %zu characters\n", >>> sizeof(c->gateway_buffer)); >>> + ret = AVERROR(EINVAL); >>> + goto err; >>> + } >>> + >>> + } >>> + >>> + // Copy the fully composed gateway path into ipfs_gateway_file. >>> + if (snprintf(ipfs_gateway_file, sizeof(ipfs_gateway_file), >>> "%sgateway", >>> + ipfs_full_data_folder) >= sizeof(ipfs_gateway_file)) { >>> + av_log(h, AV_LOG_WARNING, "The IPFS gateway file path exceeds >>> the max path length (%zu)\n", sizeof(ipfs_gateway_file)); >>> + ret = AVERROR(ENOENT); >>> + goto err; >>> + } >>> + >>> + // Get the contents of the gateway file. >>> + gateway_file = av_fopen_utf8(ipfs_gateway_file, "r"); >>> + if (!gateway_file) { >>> + av_log(h, AV_LOG_WARNING, "The IPFS gateway file (full uri: %s) >>> doesn't exist. Is the gateway enabled?\n", ipfs_gateway_file); >>> + ret = AVERROR(ENOENT); >>> + goto err; >>> + } >>> + >>> + // Read a single line (fgets stops at new line mark). >>> + fgets(c->gateway_buffer, sizeof(c->gateway_buffer) - 1, >>> gateway_file); >>> + >>> + // Replace the last char with \0 >>> + c->gateway_buffer[sizeof(c->gateway_buffer) - 1] = 0; >>> + >>> + // Replace first occurence of end of line with \0 >>> + c->gateway_buffer[strcspn(c->gateway_buffer, "\r")] = 0; >>> + c->gateway_buffer[strcspn(c->gateway_buffer, "\n")] = 0; >>> + >>> + // If strlen finds anything longer then 0 characters then we have a >>> + // potential gateway url. >>> + if (strlen(c->gateway_buffer) < 1) { >>> + av_log(h, AV_LOG_WARNING, "The IPFS gateway file (full uri: %s) >>> appears to be empty. Is the gateway started?\n", ipfs_gateway_file); >>> + ret = AVERROR(EILSEQ); >>> + goto err; >>> + } else { >>> + // We're done, the c->gateway_buffer has something that looks >>> valid. >>> + ret = 1; >>> + goto err; >>> + } >>> + >>> +err: >>> + if (gateway_file) >>> + fclose(gateway_file); >>> + >>> + return ret; >>> +} >>> + >>> +static int translate_ipfs_to_http(URLContext *h, const char *uri, >>> + int flags, AVDictionary **options) >>> +{ >>> + const char *ipfs_cid; >>> + char *fulluri = NULL; >>> + int ret; >>> + IPFSGatewayContext *c = h->priv_data; >>> + >>> + // Test for ipfs://, ipfs:, ipns:// and ipns:. This prefix is >>> stripped from >>> + // the string leaving just the CID in ipfs_cid. >>> + int is_ipfs = av_stristart(uri, "ipfs://", &ipfs_cid); >>> + int is_ipns = av_stristart(uri, "ipns://", &ipfs_cid); >>> + >>> + // We must have either ipns or ipfs. >>> + if (!is_ipfs && !is_ipns) { >>> + ret = AVERROR(EINVAL); >>> + av_log(h, AV_LOG_WARNING, "Unsupported url %s\n", uri); >>> + goto err; >>> + } >>> + >>> + // If the CID has a length greater then 0 then we assume we have a >>> proper working one. >>> + // It could still be wrong but in that case the gateway should save >>> us and >>> + // ruturn a 403 error. The http protocol handles this. >>> + if (strlen(ipfs_cid) < 1) { >>> + av_log(h, AV_LOG_WARNING, "A CID must be provided.\n"); >>> + ret = AVERROR(EILSEQ); >>> + goto err; >>> + } >>> + >>> + // Populate c->gateway_buffer with whatever is in c->gateway >>> + if (c->gateway != NULL) { >>> + if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s", >>> + c->gateway) >= sizeof(c->gateway_buffer)) { >>> + av_log(h, AV_LOG_WARNING, "The -gateway parameter is too >>> long. We allow a max of %zu characters\n", sizeof(c->gateway_buffer)); >>> + ret = AVERROR(EINVAL); >>> + goto err; >>> + } >>> + } else { >>> + // Populate the IPFS gateway if we have any. >>> + // If not, inform the user how to properly set one. >>> + ret = populate_ipfs_gateway(h); >>> + >>> + if (ret < 1) { >>> + // We fallback on dweb.link (managed by Protocol Labs). >>> + snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), " >>> https://dweb.link"); >>> >> >> Just to make this part clear. >> I've been holding off sending this patch till I received and OK from >> Protocol Labs (they maintain this gateway and IPFS). >> They approve having this fallback gateway in the ffmpeg codebase. >> >> This also means that, starting from this patch, "ffplay ipfs://<cid>" >> will just work on any platform where ffmpeg runs and this protocol is >> enabled! >> That's huge! >> >> Other than that it's implemented exactly as previously discussed in the >> V8 series. >> >> + >>> + av_log(h, AV_LOG_WARNING, "IPFS does not appear to be >>> running. You’re now using the public gateway at dweb.link.\n"); >>> + av_log(h, AV_LOG_INFO, "Installing IPFS locally is >>> recommended to improve performance and reliability, and not share all your >>> activity with a single IPFS gateway.\n"); >>> + av_log(h, AV_LOG_INFO, "There are multiple options to >>> define this gateway.\n"); >>> + av_log(h, AV_LOG_INFO, "1. Call ffmpeg with a gateway >>> param, without a trailing slash: -gateway <url>.\n"); >>> + av_log(h, AV_LOG_INFO, "2. Define an $IPFS_GATEWAY >>> environment variable with the full HTTP URL to the gateway without trailing >>> forward slash.\n"); >>> + av_log(h, AV_LOG_INFO, "3. Define an $IPFS_PATH environment >>> variable and point it to the IPFS data path - this is typically ~/.ipfs\n"); >>> + } >>> + } >>> + >>> + // Test if the gateway starts with either http:// or https:// >>> + if (av_stristart(c->gateway_buffer, "http://", NULL) == 0 >>> + && av_stristart(c->gateway_buffer, "https://", NULL) == 0) { >>> + av_log(h, AV_LOG_WARNING, "The gateway URL didn't start with >>> http:// or https:// and is therefore invalid.\n"); >>> + ret = AVERROR(EILSEQ); >>> + goto err; >>> + } >>> + >>> + // Concatenate the url. >>> + // This ends up with something like: >>> http://localhost:8080/ipfs/Qm..... >>> + // The format of "%s%s%s%s" is the following: >>> + // 1st %s = The gateway. >>> + // 2nd %s = If the gateway didn't end in a slash, add a "/". >>> Otherwise it's an empty string >>> + // 3rd %s = Either ipns/ or ipfs/. >>> + // 4th %s = The IPFS CID (Qm..., bafy..., ...). >>> + fulluri = av_asprintf("%s%s%s%s", >>> + c->gateway_buffer, >>> + (c->gateway_buffer[strlen(c->gateway_buffer) >>> - 1] == '/') ? "" : "/", >>> + (is_ipns) ? "ipns/" : "ipfs/", >>> + ipfs_cid); >>> + >>> + // Pass the URL back to FFMpeg's protocol handler. >>> + if ((ret = ffurl_open_whitelist(&c->inner, fulluri, flags, >>> + &h->interrupt_callback, options, >>> + h->protocol_whitelist, >>> + h->protocol_blacklist, h)) >>> + < 0) { >>> + av_log(h, AV_LOG_WARNING, "Unable to open resource: %s\n", >>> fulluri); >>> + goto err; >>> + } >>> + >>> +err: >>> + av_free(fulluri); >>> + return ret; >>> +} >>> + >>> +static int ipfs_read(URLContext *h, unsigned char *buf, int size) >>> +{ >>> + IPFSGatewayContext *c = h->priv_data; >>> + return ffurl_read(c->inner, buf, size); >>> +} >>> + >>> +static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence) >>> +{ >>> + IPFSGatewayContext *c = h->priv_data; >>> + return ffurl_seek(c->inner, pos, whence); >>> +} >>> + >>> +static int ipfs_close(URLContext *h) >>> +{ >>> + IPFSGatewayContext *c = h->priv_data; >>> + av_free(c->gateway); >>> + return ffurl_closep(&c->inner); >>> +} >>> + >>> +#define OFFSET(x) offsetof(IPFSGatewayContext, x) >>> + >>> +static const AVOption options[] = { >>> + {"gateway", "The gateway to ask for IPFS data.", OFFSET(gateway), >>> AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM}, >>> + {NULL}, >>> +}; >>> + >>> +static const AVClass ipfs_context_class = { >>> + .class_name = "IPFS", >>> + .item_name = av_default_item_name, >>> + .option = options, >>> + .version = LIBAVUTIL_VERSION_INT, >>> +}; >>> + >>> +const URLProtocol ff_ipfs_protocol = { >>> + .name = "ipfs", >>> + .url_open2 = translate_ipfs_to_http, >>> + .url_read = ipfs_read, >>> + .url_seek = ipfs_seek, >>> + .url_close = ipfs_close, >>> + .priv_data_size = sizeof(IPFSGatewayContext), >>> + .priv_data_class = &ipfs_context_class, >>> +}; >>> + >>> +const URLProtocol ff_ipns_protocol = { >>> + .name = "ipns", >>> + .url_open2 = translate_ipfs_to_http, >>> + .url_read = ipfs_read, >>> + .url_seek = ipfs_seek, >>> + .url_close = ipfs_close, >>> + .priv_data_size = sizeof(IPFSGatewayContext), >>> + .priv_data_class = &ipfs_context_class, >>> +}; >>> diff --git a/libavformat/protocols.c b/libavformat/protocols.c >>> index 948fae411f..675b684bd3 100644 >>> --- a/libavformat/protocols.c >>> +++ b/libavformat/protocols.c >>> @@ -73,6 +73,8 @@ extern const URLProtocol ff_libsrt_protocol; >>> extern const URLProtocol ff_libssh_protocol; >>> extern const URLProtocol ff_libsmbclient_protocol; >>> extern const URLProtocol ff_libzmq_protocol; >>> +extern const URLProtocol ff_ipfs_protocol; >>> +extern const URLProtocol ff_ipns_protocol; >>> >>> #include "libavformat/protocol_list.c" >>> >>> -- >>> 2.35.1 >>> >>> > ping... > 5 days later. Second ping.. _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".