On Thu, Feb 17, 2022 at 3:57 PM Mark Gaiser <mark...@gmail.com> wrote:
> On Thu, Feb 17, 2022 at 3:50 PM Mark Gaiser <mark...@gmail.com> wrote: > >> This patch adds support for: >> - ffplay ipfs://<cid> >> - ffplay ipns://<cid> >> >> IPFS data can be played from so called "ipfs gateways". >> A gateway is essentially a webserver that gives access to the >> distributed IPFS network. >> >> This protocol support (ipfs and ipns) therefore translates >> ipfs:// and ipns:// to a http:// url. This resulting url is >> then handled by the http protocol. It could also be https >> depending on the gateway provided. >> >> To use this protocol, a gateway must be provided. >> If you do nothing it will try to find it in your >> $HOME/.ipfs/gateway file. The ways to set it manually are: >> 1. Define a -gateway <url> to the gateway. >> 2. Define $IPFS_GATEWAY with the full http link to the gateway. >> 3. Define $IPFS_PATH and point it to the IPFS data path. >> 4. Have IPFS running in your local user folder (under $HOME/.ipfs). >> >> Signed-off-by: Mark Gaiser <mark...@gmail.com> >> --- >> configure | 2 + >> doc/protocols.texi | 30 ++++ >> libavformat/Makefile | 2 + >> libavformat/ipfsgateway.c | 309 ++++++++++++++++++++++++++++++++++++++ >> libavformat/protocols.c | 2 + >> 5 files changed, 345 insertions(+) >> create mode 100644 libavformat/ipfsgateway.c >> >> diff --git a/configure b/configure >> index 5b19a35f59..6ff09e7974 100755 >> --- a/configure >> +++ b/configure >> @@ -3585,6 +3585,8 @@ udp_protocol_select="network" >> udplite_protocol_select="network" >> unix_protocol_deps="sys_un_h" >> unix_protocol_select="network" >> +ipfs_protocol_select="https_protocol" >> +ipns_protocol_select="https_protocol" >> >> # external library protocols >> libamqp_protocol_deps="librabbitmq" >> diff --git a/doc/protocols.texi b/doc/protocols.texi >> index d207df0b52..7c9c0a4808 100644 >> --- a/doc/protocols.texi >> +++ b/doc/protocols.texi >> @@ -2025,5 +2025,35 @@ decoding errors. >> >> @end table >> >> +@section ipfs >> + >> +InterPlanetary File System (IPFS) protocol support. One can access files >> stored >> +on the IPFS network through so called gateways. Those are http(s) >> endpoints. >> +This protocol wraps the IPFS native protocols (ipfs:// and ipns://) to >> be send >> +to such a gateway. Users can (and should) host their own node which >> means this >> +protocol will use your local machine gateway to access files on the IPFS >> network. >> + >> +If a user doesn't have a node of their own then the public gateway >> dweb.link is >> +used by default. >> + >> +You can use this protocol in 2 ways. Using IPFS: >> +@example >> +ffplay ipfs://QmbGtJg23skhvFmu9mJiePVByhfzu5rwo74MEkVDYAmF5T >> +@end example >> + >> +Or the IPNS protocol (IPNS is mutable IPFS): >> +@example >> +ffplay ipns://QmbGtJg23skhvFmu9mJiePVByhfzu5rwo74MEkVDYAmF5T >> +@end example >> + >> +You can also change the gateway to be used: >> + >> +@table @option >> + >> +@item gateway >> +Defines the gateway to use. When nothing is provided the protocol will >> first try >> +your local gateway. If that fails dweb.link will be used. >> + >> +@end table >> >> @c man end PROTOCOLS >> diff --git a/libavformat/Makefile b/libavformat/Makefile >> index 3dc6a479cc..4edce8420f 100644 >> --- a/libavformat/Makefile >> +++ b/libavformat/Makefile >> @@ -656,6 +656,8 @@ OBJS-$(CONFIG_SRTP_PROTOCOL) += >> srtpproto.o srtp.o >> OBJS-$(CONFIG_SUBFILE_PROTOCOL) += subfile.o >> OBJS-$(CONFIG_TEE_PROTOCOL) += teeproto.o tee_common.o >> OBJS-$(CONFIG_TCP_PROTOCOL) += tcp.o >> +OBJS-$(CONFIG_IPFS_PROTOCOL) += ipfsgateway.o >> +OBJS-$(CONFIG_IPNS_PROTOCOL) += ipfsgateway.o >> TLS-OBJS-$(CONFIG_GNUTLS) += tls_gnutls.o >> TLS-OBJS-$(CONFIG_LIBTLS) += tls_libtls.o >> TLS-OBJS-$(CONFIG_MBEDTLS) += tls_mbedtls.o >> diff --git a/libavformat/ipfsgateway.c b/libavformat/ipfsgateway.c >> new file mode 100644 >> index 0000000000..7dfa56871d >> --- /dev/null >> +++ b/libavformat/ipfsgateway.c >> @@ -0,0 +1,309 @@ >> +/* >> + * IPFS and IPNS protocol support through IPFS Gateway. >> + * Copyright (c) 2022 Mark Gaiser >> + * >> + * This file is part of FFmpeg. >> + * >> + * FFmpeg is free software; you can redistribute it and/or >> + * modify it under the terms of the GNU Lesser General Public >> + * License as published by the Free Software Foundation; either >> + * version 2.1 of the License, or (at your option) any later version. >> + * >> + * FFmpeg is distributed in the hope that it will be useful, >> + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU >> + * Lesser General Public License for more details. >> + * >> + * You should have received a copy of the GNU Lesser General Public >> + * License along with FFmpeg; if not, write to the Free Software >> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA >> 02110-1301 USA >> + */ >> + >> +#include "avformat.h" >> +#include "libavutil/avassert.h" >> +#include "libavutil/avstring.h" >> +#include "libavutil/internal.h" >> +#include "libavutil/opt.h" >> +#include "libavutil/tree.h" >> +#include <fcntl.h> >> +#if HAVE_IO_H >> +#include <io.h> >> +#endif >> +#if HAVE_UNISTD_H >> +#include <unistd.h> >> +#endif >> +#include "os_support.h" >> +#include "url.h" >> +#include <stdlib.h> >> +#include <sys/stat.h> >> + >> +typedef struct IPFSGatewayContext { >> + AVClass *class; >> + URLContext *inner; >> + // Is filled by the -gateway argument and not changed after. >> + char *gateway; >> + // If the above gateway is non null, it will be copied into this >> buffer. >> + // Else this buffer will contain the auto detected gateway. >> + // In either case, the gateway to use will be in this buffer. >> + char gateway_buffer[PATH_MAX]; >> +} IPFSGatewayContext; >> + >> +// A best-effort way to find the IPFS gateway. >> +// Only the most appropiate gateway is set. It's not actually requested >> +// (http call) to prevent a potential slowdown in startup. A potential >> timeout >> +// is handled by the HTTP protocol. >> +static int populate_ipfs_gateway(URLContext *h) >> +{ >> + IPFSGatewayContext *c = h->priv_data; >> + char ipfs_full_data_folder[PATH_MAX]; >> + char ipfs_gateway_file[PATH_MAX]; >> + struct stat st; >> + int stat_ret = 0; >> + int ret = AVERROR(EINVAL); >> + FILE *gateway_file = NULL; >> + >> + // Test $IPFS_GATEWAY. >> + if (getenv("IPFS_GATEWAY") != NULL) { >> + if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s", >> + getenv("IPFS_GATEWAY")) >= >> sizeof(c->gateway_buffer)) { >> + av_log(h, AV_LOG_ERROR, "The IPFS_GATEWAY environment >> variable exceeds the maximum length. We allow a max of %zu characters\n", >> sizeof(c->gateway_buffer)); >> + ret = AVERROR(EINVAL); >> + goto err; >> + } >> + >> + ret = 1; >> + goto err; >> + } else >> + av_log(h, AV_LOG_DEBUG, "$IPFS_GATEWAY is empty.\n"); >> + >> + // We need to know the IPFS folder to - eventually - read the >> contents of >> + // the "gateway" file which would tell us the gateway to use. >> + if (getenv("IPFS_PATH") == NULL) { >> + av_log(h, AV_LOG_DEBUG, "$IPFS_PATH is empty.\n"); >> + >> + // Try via the home folder. >> + if (getenv("HOME") == NULL) { >> + av_log(h, AV_LOG_ERROR, "$HOME appears to be empty.\n"); >> + ret = AVERROR(EINVAL); >> + goto err; >> + } >> + >> + // Verify the composed path fits. >> + if (snprintf(ipfs_full_data_folder, >> sizeof(ipfs_full_data_folder), >> + "%s/.ipfs/", getenv("HOME")) >= >> sizeof(ipfs_full_data_folder)) { >> + av_log(h, AV_LOG_ERROR, "The IPFS data path exceeds the max >> path length (%zu)\n", sizeof(ipfs_full_data_folder)); >> + ret = AVERROR(EINVAL); >> + goto err; >> + } >> + >> + // Stat the folder. >> + // It should exist in a default IPFS setup when run as local >> user. >> +#ifndef _WIN32 >> + stat_ret = stat(ipfs_full_data_folder, &st); >> +#else >> + stat_ret = win32_stat(ipfs_full_data_folder, &st); >> +#endif >> + if (stat_ret < 0) { >> + av_log(h, AV_LOG_INFO, "Unable to find IPFS folder. We >> tried:\n"); >> + av_log(h, AV_LOG_INFO, "- $IPFS_PATH, which was empty.\n"); >> + av_log(h, AV_LOG_INFO, "- $HOME/.ipfs (full uri: %s) which >> doesn't exist.\n", ipfs_full_data_folder); >> + ret = AVERROR(ENOENT); >> + goto err; >> + } >> + } else { >> + if (snprintf(ipfs_full_data_folder, >> sizeof(ipfs_full_data_folder), "%s", >> + getenv("IPFS_PATH")) >= sizeof(ipfs_full_data_folder)) { >> + av_log(h, AV_LOG_ERROR, "The IPFS_PATH environment variable >> exceeds the maximum length. We allow a max of %zu characters\n", >> sizeof(c->gateway_buffer)); >> + ret = AVERROR(EINVAL); >> + goto err; >> + } >> + >> + } >> + >> + // Copy the fully composed gateway path into ipfs_gateway_file. >> + if (snprintf(ipfs_gateway_file, sizeof(ipfs_gateway_file), >> "%sgateway", >> + ipfs_full_data_folder) >= sizeof(ipfs_gateway_file)) { >> + av_log(h, AV_LOG_ERROR, "The IPFS gateway file path exceeds the >> max path length (%zu)\n", sizeof(ipfs_gateway_file)); >> + ret = AVERROR(ENOENT); >> + goto err; >> + } >> + >> + // Get the contents of the gateway file. >> + gateway_file = av_fopen_utf8(ipfs_gateway_file, "r"); >> + if (!gateway_file) { >> + av_log(h, AV_LOG_ERROR, "The IPFS gateway file (full uri: %s) >> doesn't exist. Is the gateway enabled?\n", ipfs_gateway_file); >> + ret = AVERROR(ENOENT); >> + goto err; >> + } >> + >> + // Read a single line (fgets stops at new line mark). >> + fgets(c->gateway_buffer, sizeof(c->gateway_buffer) - 1, >> gateway_file); >> + >> + // Replace the last char with \0 >> + c->gateway_buffer[sizeof(c->gateway_buffer) - 1] = 0; >> + >> + // Replace first occurence of end of line with \0 >> + c->gateway_buffer[strcspn(c->gateway_buffer, "\r")] = 0; >> + c->gateway_buffer[strcspn(c->gateway_buffer, "\n")] = 0; >> + >> + // If strlen finds anything longer then 0 characters then we have a >> + // potential gateway url. >> + if (strlen(c->gateway_buffer) < 1) { >> + av_log(h, AV_LOG_ERROR, "The IPFS gateway file (full uri: %s) >> appears to be empty. Is the gateway started?\n", ipfs_gateway_file); >> + ret = AVERROR(EILSEQ); >> + goto err; >> + } else { >> + // We're done, the c->gateway_buffer has something that looks >> valid. >> + ret = 1; >> + goto err; >> + } >> + >> +err: >> + if (gateway_file) >> + fclose(gateway_file); >> + >> + return ret; >> +} >> + >> +static int translate_ipfs_to_http(URLContext *h, const char *uri, >> + int flags, AVDictionary **options) >> +{ >> + const char *ipfs_cid; >> + char *fulluri = NULL; >> + int ret; >> + IPFSGatewayContext *c = h->priv_data; >> + >> + // Test for ipfs://, ipfs:, ipns:// and ipns:. This prefix is >> stripped from >> + // the string leaving just the CID in ipfs_cid. >> + int is_ipfs = av_stristart(uri, "ipfs://", &ipfs_cid); >> + int is_ipns = av_stristart(uri, "ipns://", &ipfs_cid); >> + >> + // We must have either ipns or ipfs. >> + if (!is_ipfs && !is_ipns) { >> + ret = AVERROR(EINVAL); >> + av_log(h, AV_LOG_ERROR, "Unsupported url %s\n", uri); >> + goto err; >> + } >> + >> + // If the CID has a length greater then 0 then we assume we have a >> proper working one. >> + // It could still be wrong but in that case the gateway should save >> us and >> + // ruturn a 403 error. The http protocol handles this. >> + if (strlen(ipfs_cid) < 1) { >> + av_log(h, AV_LOG_ERROR, "A CID must be provided.\n"); >> + ret = AVERROR(EILSEQ); >> + goto err; >> + } >> + >> + // Populate c->gateway_buffer with whatever is in c->gateway >> + if (c->gateway != NULL) { >> + if (snprintf(c->gateway_buffer, sizeof(c->gateway_buffer), "%s", >> + c->gateway) >= sizeof(c->gateway_buffer)) { >> + av_log(h, AV_LOG_ERROR, "The -gateway parameter is too long. >> We allow a max of %zu characters\n", sizeof(c->gateway_buffer)); >> + ret = AVERROR(EINVAL); >> + goto err; >> + } >> + } else { >> + // Populate the IPFS gateway if we have any. >> + // If not, inform the user how to properly set one. >> + ret = populate_ipfs_gateway(h); >> + >> + if (ret < 1) { >> + av_log(h, AV_LOG_ERROR, "No IPFS gateway was set. Make sure >> a local IPFS instance is running.\n"); >> + av_log(h, AV_LOG_INFO, "There are multiple options to define >> this gateway. The below options are in order of precedence:\n"); >> + av_log(h, AV_LOG_INFO, "1. Define a -gateway <url> to the >> gateway without trailing forward slash.\n"); >> + av_log(h, AV_LOG_INFO, "2. Define $IPFS_GATEWAY with the >> full http link to the gateway without trailing forward slash.\n"); >> + av_log(h, AV_LOG_INFO, "3. Define $IPFS_PATH and point it to >> the IPFS data path.\n"); >> + av_log(h, AV_LOG_INFO, "4. Have IPFS running in your local >> user folder (under $HOME/.ipfs).\n"); >> + av_log(h, AV_LOG_INFO, "In all path cases, a file named >> gateway is expected. See https://github.com/ipfs/specs/issues/261 for >> more information.\n"); >> + goto err; >> + } >> + } >> + >> + // Test if the gateway starts with either http:// or https:// >> + if (av_stristart(c->gateway_buffer, "http://", NULL) == 0 >> + && av_stristart(c->gateway_buffer, "https://", NULL) == 0) { >> + av_log(h, AV_LOG_ERROR, "The gateway URL didn't start with >> http:// or https:// and is therefore invalid.\n"); >> + ret = AVERROR(EILSEQ); >> + goto err; >> + } >> + >> + // Concatenate the url. >> + // This ends up with something like: >> http://localhost:8080/ipfs/Qm..... >> + // The format of "%s%s%s%s" is the following: >> + // 1st %s = The gateway. >> + // 2nd %s = If the gateway didn't end in a slash, add a "/". >> Otherwise it's an empty string >> + // 3rd %s = Either ipns/ or ipfs/. >> + // 4th %s = The IPFS CID (Qm..., bafy..., ...). >> + fulluri = av_asprintf("%s%s%s%s", >> + c->gateway_buffer, >> + (c->gateway_buffer[strlen(c->gateway_buffer) - >> 1] == '/') ? "" : "/", >> + (is_ipns) ? "ipns/" : "ipfs/", >> + ipfs_cid); >> + >> + // Pass the URL back to FFMpeg's protocol handler. >> + if ((ret = ffurl_open_whitelist(&c->inner, fulluri, flags, >> + &h->interrupt_callback, options, >> + h->protocol_whitelist, >> + h->protocol_blacklist, h)) >> + < 0) { >> + av_log(h, AV_LOG_ERROR, "Unable to open resource: %s\n", >> fulluri); >> + goto err; >> + } >> + >> +err: >> + av_free(fulluri); >> + return ret; >> +} >> + >> +static int ipfs_read(URLContext *h, unsigned char *buf, int size) >> +{ >> + IPFSGatewayContext *c = h->priv_data; >> + return ffurl_read(c->inner, buf, size); >> +} >> + >> +static int64_t ipfs_seek(URLContext *h, int64_t pos, int whence) >> +{ >> + IPFSGatewayContext *c = h->priv_data; >> + return ffurl_seek(c->inner, pos, whence); >> +} >> + >> +static int ipfs_close(URLContext *h) >> +{ >> + IPFSGatewayContext *c = h->priv_data; >> + av_free(c->gateway); >> + return ffurl_closep(&c->inner); >> +} >> + >> +#define OFFSET(x) offsetof(IPFSGatewayContext, x) >> + >> +static const AVOption options[] = { >> + {"gateway", "The gateway to ask for IPFS data.", OFFSET(gateway), >> AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM}, >> + {NULL}, >> +}; >> + >> +static const AVClass ipfs_context_class = { >> + .class_name = "IPFS", >> + .item_name = av_default_item_name, >> + .option = options, >> + .version = LIBAVUTIL_VERSION_INT, >> +}; >> + >> +const URLProtocol ff_ipfs_protocol = { >> + .name = "ipfs", >> + .url_open2 = translate_ipfs_to_http, >> + .url_read = ipfs_read, >> + .url_seek = ipfs_seek, >> + .url_close = ipfs_close, >> + .priv_data_size = sizeof(IPFSGatewayContext), >> + .priv_data_class = &ipfs_context_class, >> +}; >> + >> +const URLProtocol ff_ipns_protocol = { >> + .name = "ipns", >> + .url_open2 = translate_ipfs_to_http, >> + .url_read = ipfs_read, >> + .url_seek = ipfs_seek, >> + .url_close = ipfs_close, >> + .priv_data_size = sizeof(IPFSGatewayContext), >> + .priv_data_class = &ipfs_context_class, >> +}; >> diff --git a/libavformat/protocols.c b/libavformat/protocols.c >> index 948fae411f..675b684bd3 100644 >> --- a/libavformat/protocols.c >> +++ b/libavformat/protocols.c >> @@ -73,6 +73,8 @@ extern const URLProtocol ff_libsrt_protocol; >> extern const URLProtocol ff_libssh_protocol; >> extern const URLProtocol ff_libsmbclient_protocol; >> extern const URLProtocol ff_libzmq_protocol; >> +extern const URLProtocol ff_ipfs_protocol; >> +extern const URLProtocol ff_ipns_protocol; >> >> #include "libavformat/protocol_list.c" >> >> -- >> 2.35.1 >> >> > There we go, V8. Would this be the final one? I hope so! :) > Just in case this is the magic final version that is allowed to be merged. > How do I proceed in that case? I don't have merge powers... > > A small note on the log lines. > I do prefer to keep them on one line. This seems to be in line with other > ffmpeg code files (i only looked at a couple other protocols like crypto, > http and avio). > Splitting them looks - imho - quite ugly. > ping _______________________________________________ ffmpeg-devel mailing list ffmpeg-devel@ffmpeg.org https://ffmpeg.org/mailman/listinfo/ffmpeg-devel To unsubscribe, visit link above, or email ffmpeg-devel-requ...@ffmpeg.org with subject "unsubscribe".