MonetDB: makelibstreamgreatagain - Let urlstream fail if http se...

[Joeri van Ruth]

Changeset: 6174453f736c for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=6174453f736c
Modified Files:
        common/stream/Tests/urlstream.py
        common/stream/url_stream.c
Branch: makelibstreamgreatagain
Log Message:

Let urlstream fail if http server sends error status


diffs (81 lines):

diff --git a/common/stream/Tests/urlstream.py b/common/stream/Tests/urlstream.py
--- a/common/stream/Tests/urlstream.py
+++ b/common/stream/Tests/urlstream.py
@@ -35,11 +35,18 @@ class Handler(http.server.BaseHTTPReques
             self.end_headers()
             self.wfile.write(b'a' * 40 * 1024)
         elif self.path == '/sleep':
+            # used to test if subprocess timeout= option works
             time.sleep(60)
             self.send_response(http.HTTPStatus.OK)
             self.send_header('Content-Type', 'text/plain')
             self.end_headers()
             self.wfile.write(b"now i'm well rested")
+        elif self.path == '/xyzzy':
+            msg = 'A hollow voice says "Fool."\n'
+            self.send_response(http.HTTPStatus.NOT_FOUND, msg)
+            self.send_header('Content-Type', 'text/plain')
+            self.end_headers()
+            self.wfile.write(bytes(msg, 'ascii'))
         else:
             self.send_response(http.HTTPStatus.NOT_FOUND)
             self.send_header('Content-Type', 'text/plain')
@@ -64,7 +71,7 @@ def streamcat(suffix):
     cmd = ['streamcat', 'read', u, 'urlstream']
     print(f'FETCHING {suffix}')
     PIPE = subprocess.PIPE
-    p = subprocess.run(cmd, check=False, stdout=PIPE, stderr=PIPE)
+    p = subprocess.run(cmd, check=False, stdout=PIPE, stderr=PIPE, timeout=10)
     return (p.returncode, p.stdout, p.stderr)
 
 
@@ -78,3 +85,12 @@ assert code == 0
 assert out == 40 * 1024 * b'a'
 assert err == b''
 
+# Are we able to time out?
+# (code, out, err) = streamcat('/sleep')
+
+(code, out, err) = streamcat('/xyzzy')
+assert code != 0
+assert b'hollow voice' in err
+
+if __name__ == "__main__":
+    input('banana>')
diff --git a/common/stream/url_stream.c b/common/stream/url_stream.c
--- a/common/stream/url_stream.c
+++ b/common/stream/url_stream.c
@@ -27,6 +27,7 @@ struct curl_data {
        size_t usesize;         /* end of used data */
        size_t offset;          /* start of unread data */
        int running;            /* whether still transferring */
+       char errbuf[CURL_ERROR_SIZE]; /* a place for error messages */
 };
 
 #define BLOCK_CURL     ((size_t) 1 << 16)
@@ -151,6 +152,7 @@ open_urlstream(const char *url)
        }
        *c = (struct curl_data) {
                .running = 1,
+               .errbuf = {0},
        };
        if ((s = create_stream(url)) == NULL) {
                free(c);
@@ -171,11 +173,16 @@ open_urlstream(const char *url)
        curl_easy_setopt(c->handle, CURLOPT_WRITEDATA, s);
        curl_easy_setopt(c->handle, CURLOPT_VERBOSE, 0);
        curl_easy_setopt(c->handle, CURLOPT_NOSIGNAL, 1);
+       curl_easy_setopt(c->handle, CURLOPT_FAILONERROR, 1);
+       curl_easy_setopt(c->handle, CURLOPT_ERRORBUFFER, c->errbuf);
        curl_easy_setopt(c->handle, CURLOPT_WRITEFUNCTION, write_callback);
        CURLcode ret = curl_easy_perform(c->handle);
        if (ret != CURLE_OK) {
+               if (strlen(c->errbuf) > 0)
+                       mnstr_set_open_error(url, 0, "%s", c->errbuf);
+               else
+                       mnstr_set_open_error(url, 0, "curl_easy_perform: %s", 
curl_easy_strerror(ret));
                curl_destroy(s);
-               mnstr_set_open_error(url, 0, "curl_easy_perform: %s", 
curl_easy_strerror(ret));
                return NULL;
        }
        curl_easy_cleanup(c->handle);
_______________________________________________
checkin-list mailing list
checkin-list@monetdb.org
https://www.monetdb.org/mailman/listinfo/checkin-list