Hello, The attached patch dumps the http headers retrieved from the debuginfod server which a file was downloaded from. Some custom headers are now returned such as X-FILE, X-FILE-SIZE, and X-ARCHIVE to give some context about the downloaded files.
Noah Sanci
From dcbe8672d6be30f92ad8baa2fa157b9b7a551b48 Mon Sep 17 00:00:00 2001 From: Noah Sanci <nsa...@redhat.com> Date: Wed, 28 Jul 2021 14:46:05 -0400 Subject: [PATCH] debuginfod: PR27277 - Describe retrieved files when verbose There appear to exist use cases that intend to simply check for the existence of content in a debuginfod server, without actually downloading it. In HTTP land, the HEAD operation is the natural expression of this. We could support this in the webapi, and give options to debuginfod-find and the client API to use it. Instead of implementing a describe option, allow users, with enough verbosity, to get a file description printed to the verbose output stream upon retrieving a file. The HEAD operation is not supported in this patch. E.g output: HTTP/1.1 200 OK Connection: Keep-Alive Content-Length: 2428240 Cache-Control: public Last-Modified: Sat, 15 May 2021 20:49:51 GMT X-FILE-SIZE: 656906321 X-FILE: /usr/lib/debug/lib/modules/5.11.21-200.fc33.x86_64/kernel/drivers/scsi/bnx2fc/bnx2fc.ko.debug X-ARCHIVE: kernel-debuginfo-5.11.21-200.fc33.x86_64.rpm Content-Type: application/octet-stream Date: Tue, 03 Aug 2021 18:50:36 GMT https://sourceware.org/bugzilla/show_bug.cgi?id=27277 Signed-off-by: Noah Sanci <nsa...@redhat.com> --- debuginfod/ChangeLog | 16 ++++++++++ debuginfod/debuginfod-client.c | 58 ++++++++++++++++++++++++++++++++-- debuginfod/debuginfod-find.c | 3 ++ debuginfod/debuginfod.cxx | 11 +++++++ debuginfod/debuginfod.h.in | 3 ++ debuginfod/libdebuginfod.map | 3 ++ doc/ChangeLog | 6 ++++ doc/debuginfod-find.1 | 3 +- tests/ChangeLog | 6 ++++ tests/run-debuginfod-find.sh | 19 ++++++++++- 10 files changed, 124 insertions(+), 4 deletions(-) diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog index 9e82d78d..4297b088 100644 --- a/debuginfod/ChangeLog +++ b/debuginfod/ChangeLog @@ -1,3 +1,19 @@ +2021-08-02 Noah Sanci <nsa...@redhat.com> + + PR27277 + * debuginfod-client.c (debuginfod_client): New field winning_headers. + (handle_data): New field response_data. + (header_callback): Store received headers in response_data. + (debuginfod_query_server): Activate CURLOPT_HEADERFUNCTION. + Save winning response_data. + (debuginfod_get_response_headers): Return the winning headers. + * debuginfod.h.in: Declare new API function. + * libdebuginfod.map: Export it. + * debuginfod-find.c: Call it in verbose mode. + * debuginfod.cxx (handle_buildid_f_match): Add X-FILE and X-FILE-SIZE + http headers to response. + (handle_buildid_r_match): Add X-FILE, X-FILE-SIZE and X-ARCHIVE headers. + 2021-07-26 Noah Sanci <nsa...@redhat.com> PR27982 diff --git a/debuginfod/debuginfod-client.c b/debuginfod/debuginfod-client.c index 7d4b220f..5c2a646c 100644 --- a/debuginfod/debuginfod-client.c +++ b/debuginfod/debuginfod-client.c @@ -127,6 +127,7 @@ struct debuginfod_client timeout or other info gotten from environment variables, the handle data, etc. So those don't have to be reparsed and recreated on each request. */ + char * winning_headers; }; /* The cache_clean_interval_s file within the debuginfod cache specifies @@ -183,6 +184,8 @@ struct handle_data to the cache. Used to ensure that a file is not downloaded from multiple servers unnecessarily. */ CURL **target_handle; + /* Response http headers for this client handle, sent from the server */ + char *response_data; }; static size_t @@ -499,6 +502,33 @@ default_progressfn (debuginfod_client *c, long a, long b) } +static size_t +header_callback (char * buffer, size_t size, size_t numitems, void * userdata) +{ + if (size != 1) + return 0; + /* Temporary buffer for realloc */ + char *temp = NULL; + size_t userlen = 0; + if (*(char**)userdata == NULL) + { + temp = malloc(numitems+1); + if (temp == NULL) + return 0; + memset(temp, '\0', numitems+1); + } + else + { + userlen = strlen(*(char**)userdata); + temp = realloc(*(char**)userdata, userlen + numitems + 1); + if (temp == NULL) + return 0; + } + strncat(temp, buffer, numitems); + *(char**)userdata = temp; + return numitems; +} + /* Query each of the server URLs found in $DEBUGINFOD_URLS for the file with the specified build-id, type (debuginfo, executable or source) and filename. filename may be NULL. If found, return a file @@ -936,10 +966,13 @@ debuginfod_query_server (debuginfod_client *c, curl_easy_setopt (data[i].handle, CURLOPT_LOW_SPEED_LIMIT, 100 * 1024L); } + data[i].response_data = NULL; curl_easy_setopt(data[i].handle, CURLOPT_FILETIME, (long) 1); curl_easy_setopt(data[i].handle, CURLOPT_FOLLOWLOCATION, (long) 1); curl_easy_setopt(data[i].handle, CURLOPT_FAILONERROR, (long) 1); curl_easy_setopt(data[i].handle, CURLOPT_NOSIGNAL, (long) 1); + curl_easy_setopt(data[i].handle, CURLOPT_HEADERFUNCTION, header_callback); + curl_easy_setopt(data[i].handle, CURLOPT_HEADERDATA, (void *) &(data[i].response_data)); #if LIBCURL_VERSION_NUM >= 0x072a00 /* 7.42.0 */ curl_easy_setopt(data[i].handle, CURLOPT_PATH_AS_IS, (long) 1); #else @@ -1161,10 +1194,10 @@ debuginfod_query_server (debuginfod_client *c, { char *effective_url = NULL; long resp_code = 500; - CURLcode ok1 = curl_easy_getinfo (target_handle, + CURLcode ok1 = curl_easy_getinfo (msg->easy_handle, CURLINFO_EFFECTIVE_URL, &effective_url); - CURLcode ok2 = curl_easy_getinfo (target_handle, + CURLcode ok2 = curl_easy_getinfo (msg->easy_handle, CURLINFO_RESPONSE_CODE, &resp_code); if(ok1 == CURLE_OK && ok2 == CURLE_OK && effective_url) @@ -1238,6 +1271,7 @@ debuginfod_query_server (debuginfod_client *c, { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ curl_easy_cleanup (data[i].handle); + free(data[i].response_data); } goto query_in_parallel; } @@ -1264,6 +1298,16 @@ debuginfod_query_server (debuginfod_client *c, tvs[0].tv_usec = tvs[1].tv_usec = 0; (void) futimes (fd, tvs); /* best effort */ + c->winning_headers = NULL; + for (int i = 0; i < num_urls; ++i) + if (data[i].handle == verified_handle && data[i].response_data != NULL) + { + c->winning_headers = data[i].response_data; + if (vfd >= 0) + dprintf(vfd, "\n%s", c->winning_headers); + data[i].response_data = NULL; + } + /* PR27571: make cache files casually unwriteable; dirs are already 0700 */ (void) fchmod(fd, 0400); @@ -1281,6 +1325,7 @@ debuginfod_query_server (debuginfod_client *c, { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ curl_easy_cleanup (data[i].handle); + free (data[i].response_data); } for (int i = 0; i < num_urls; ++i) @@ -1304,6 +1349,7 @@ debuginfod_query_server (debuginfod_client *c, { curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */ curl_easy_cleanup (data[i].handle); + free (data[i].response_data); } unlink (target_cache_tmppath); @@ -1415,6 +1461,7 @@ debuginfod_end (debuginfod_client *client) curl_multi_cleanup (client->server_mhandle); curl_slist_free_all (client->headers); + free (client->winning_headers); free (client->url); free (client); } @@ -1483,6 +1530,13 @@ debuginfod_set_progressfn(debuginfod_client *client, client->progressfn = fn; } +/* Return the http response headers that came from the winning server */ +const char * +debuginfod_get_response_headers(debuginfod_client *c) +{ + return c->winning_headers; +} + void debuginfod_set_verbose_fd(debuginfod_client *client, int fd) { diff --git a/debuginfod/debuginfod-find.c b/debuginfod/debuginfod-find.c index 3e8ab203..1ed76ffc 100644 --- a/debuginfod/debuginfod-find.c +++ b/debuginfod/debuginfod-find.c @@ -218,6 +218,9 @@ main(int argc, char** argv) const char* url = debuginfod_get_url (client); if (url != NULL) fprintf(stderr, "Downloaded from %s\n", url); + const char *headers = debuginfod_get_response_headers (client); + if (headers != NULL) + fprintf(stderr, "Headers:\n%s\n", headers); } debuginfod_end (client); diff --git a/debuginfod/debuginfod.cxx b/debuginfod/debuginfod.cxx index 4ddd9255..cc653a3a 100644 --- a/debuginfod/debuginfod.cxx +++ b/debuginfod/debuginfod.cxx @@ -1063,6 +1063,8 @@ handle_buildid_f_match (bool internal_req_t, else { MHD_add_response_header (r, "Content-Type", "application/octet-stream"); + (void) MHD_add_response_header(r, "X-FILE-SIZE", std::to_string(s.st_size).c_str()); + (void) MHD_add_response_header(r, "X-FILE", b_source0.c_str()); add_mhd_last_modified (r, s.st_mtime); if (verbose > 1) obatched(clog) << "serving file " << b_source0 << endl; @@ -1673,6 +1675,15 @@ handle_buildid_r_match (bool internal_req_p, else { MHD_add_response_header (r, "Content-Type", "application/octet-stream"); + size_t place = b_source0.find_last_of("/"); + + if (place == string::npos) + (void) MHD_add_response_header(r, "X-ARCHIVE", b_source0.c_str()); + else + (void) MHD_add_response_header(r, "X-ARCHIVE", (char *) ((unsigned long)b_source0.c_str() + place + 1) ); + + (void) MHD_add_response_header(r, "X-FILE", b_source1.c_str()); + (void) MHD_add_response_header(r, "X-FILE-SIZE", std::to_string(fs.st_size).c_str()); add_mhd_last_modified (r, archive_entry_mtime(e)); if (verbose > 1) obatched(clog) << "serving archive " << b_source0 << " file " << b_source1 << endl; diff --git a/debuginfod/debuginfod.h.in b/debuginfod/debuginfod.h.in index c358df4d..21a1bab8 100644 --- a/debuginfod/debuginfod.h.in +++ b/debuginfod/debuginfod.h.in @@ -93,6 +93,9 @@ void* debuginfod_get_user_data (debuginfod_client *client); /* Get the current or last active URL, if known. */ const char* debuginfod_get_url (debuginfod_client *client); +/* Get last set of HTTP response headers, if known. */ +const char* debuginfod_get_response_headers (debuginfod_client *client); + /* Add an outgoing HTTP request "Header: Value". Copies string. */ int debuginfod_add_http_header (debuginfod_client *client, const char* header); diff --git a/debuginfod/libdebuginfod.map b/debuginfod/libdebuginfod.map index 7d2f5882..1798ff63 100644 --- a/debuginfod/libdebuginfod.map +++ b/debuginfod/libdebuginfod.map @@ -18,3 +18,6 @@ ELFUTILS_0.179 { ELFUTILS_0.183 { debuginfod_set_verbose_fd; } ELFUTILS_0.179; +ELFUTILS_0.186 { + debuginfod_get_response_headers; +} ELFUTILS_0.183; diff --git a/doc/ChangeLog b/doc/ChangeLog index 1822fc6b..855cc4a2 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,3 +1,9 @@ +2021-08-04 Noah Sanci <nsa...@redhat.com> + + PR27277 + * debuginfod-find.1: Increasing verbosity describes the downloaded + file. + 2021-07-26 Noah Sanci <nsa...@redhat.com> PR27982 diff --git a/doc/debuginfod-find.1 b/doc/debuginfod-find.1 index 482a8ae7..9010aa7b 100644 --- a/doc/debuginfod-find.1 +++ b/doc/debuginfod-find.1 @@ -110,7 +110,8 @@ l l. .TP .B "\-v" -Increase verbosity, including printing frequent download-progress messages. +Increase verbosity, including printing frequent download-progress messages +and printing a brief description of the downloaded file. .SH "SECURITY" diff --git a/tests/ChangeLog b/tests/ChangeLog index 34666609..3ca81418 100644 --- a/tests/ChangeLog +++ b/tests/ChangeLog @@ -1,3 +1,9 @@ +2021-08-02 Noah Sanci <nsa...@redhat.com> + + PR27277 + * run-debuginfod-find.sh: Add a test to ensure that file descriptions + are accurate for files outside or within archives + 2021-07-26 Noah Sanci <nsa...@redhat.com> PR27982 diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh index 991d1dc5..710ac03f 100755 --- a/tests/run-debuginfod-find.sh +++ b/tests/run-debuginfod-find.sh @@ -526,7 +526,24 @@ wait_ready $PORT1 'fdcache_op_count{op="evict"}' $( grep -c 'evicted a=.*' vlog$ wait_ready $PORT1 'fdcache_op_count{op="prefetch_enqueue"}' $( grep -c 'interned.*front=0' vlog$PORT1 ) wait_ready $PORT1 'fdcache_op_count{op="prefetch_evict"}' $( grep -c 'evicted from prefetch a=.*front=0' vlog$PORT1 || true ) ######################################################################## - +## PR27277 +# Check to see if source file not located in an archive prints the file's description +env DEBUGINFOD_URLS="http://127.0.0.1:$PORT1"; LD_LIBRARY_PATH=$ldpath DEBUGINFOD_MASIZE=1\ + ${abs_top_builddir}/debuginfod/debuginfod-find -v source $BUILDID2 ${PWD}/prog2.c > vlog-find$PORT1.1 2>&1 + +tempfiles vlog-find$PORT1.1 +grep 'X-FILE: '$(realpath $PWD)'/prog2.c' vlog-find$PORT1.1 +grep 'X-FILE-SIZE: '$(wc -c ${PWD}'/prog2.c' | awk '{print $1}') vlog-find$PORT1.1 + +# Check to see if an executable file located in an archive prints the file's description and archive +env DEBUGINFOD_URLS="http://127.0.0.1:$PORT1"; LD_LIBRARY_PATH=$ldpath \ + ${abs_top_builddir}/debuginfod/debuginfod-find $VERBOSE executable c36708a78618d597dee15d0dc989f093ca5f9120 > vlog-find$PORT1.2 2>&1 + +tempfiles vlog-find$PORT1.2 +grep 'X-FILE: ' vlog-find$PORT1.2 +grep 'X-FILE-SIZE: ' vlog-find$PORT1.2 +grep 'X-ARCHIVE: hello2-1.0-2.x86_64.rpm' vlog-find$PORT1.2 +######################################################################## # Federation mode # find another unused port -- 2.31.1
