On Wed, 16 Jul 2014, David Meyer wrote:
I've found that curl_multi slows down quadratically with the number of running requests.
Hey again,I suggest a patch like the attached one. It'd be great if you could try this on your use case and see how it behaves!
-- / daniel.haxx.se
From 3c8c873252fa486d1e57fa28b8b455bfdb487f77 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg <dan...@haxx.se> Date: Tue, 2 Sep 2014 09:29:50 +0200 Subject: [PATCH] multi: convert CURLM_STATE_CONNECT_PEND handling to a list ... instead of scanning through all handles, stash only the actual handles that are in that state in the new ->pending list and scan that list only. It should be mostly empty or very short. And only used for pipelining. This avoids a rather hefty slow-down especially notable if you add many handles to the same multi handle. Regression introduced in commit 0f147887 (version 7.30.0). Bug: http://curl.haxx.se/mail/lib-2014-07/0206.html Reported-by: David Meyer --- lib/multi.c | 27 +++++++++++++++++++++------ lib/multihandle.h | 5 ++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/lib/multi.c b/lib/multi.c index 557be06..7947436 100644 --- a/lib/multi.c +++ b/lib/multi.c @@ -307,10 +307,14 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ multi->msglist = Curl_llist_alloc(multi_freeamsg); if(!multi->msglist) goto error; + multi->pending = Curl_llist_alloc(multi_freeamsg); + if(!multi->pending) + goto error; + /* allocate a new easy handle to use when closing cached connections */ multi->closure_handle = curl_easy_init(); if(!multi->closure_handle) goto error; @@ -332,10 +336,11 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ Curl_conncache_destroy(multi->conn_cache); multi->conn_cache = NULL; Curl_close(multi->closure_handle); multi->closure_handle = NULL; Curl_llist_destroy(multi->msglist, NULL); + Curl_llist_destroy(multi->pending, NULL); free(multi); return NULL; } @@ -1044,11 +1049,16 @@ static CURLMcode multi_runsingle(struct Curl_multi *multi, &async, &protocol_connect); if(CURLE_NO_CONNECTION_AVAILABLE == data->result) { /* There was no connection available. We will go to the pending state and wait for an available connection. */ multistate(data, CURLM_STATE_CONNECT_PEND); - data->result = CURLE_OK; + + /* add this handle to the list of connect-pending handles */ + if(!Curl_llist_insert_next(multi->pending, multi->pending->tail, data)) + data->result = CURLM_OUT_OF_MEMORY; + else + data->result = CURLE_OK; break; } if(CURLE_OK == data->result) { /* Add this handle to the send or pend pipeline */ @@ -1882,10 +1892,14 @@ CURLMcode curl_multi_cleanup(CURLM *multi_handle) /* remove the pending list of messages */ Curl_llist_destroy(multi->msglist, NULL); multi->msglist = NULL; + /* remove the pending handles queue */ + Curl_llist_destroy(multi->pending, NULL); + multi->msglist = NULL; + /* remove all easy handles */ data = multi->easyp; while(data) { nextdata=data->next; if(data->dns.hostcachetype == HCACHE_MULTI) { @@ -2774,20 +2788,21 @@ struct curl_llist *Curl_multi_pipelining_server_bl(struct Curl_multi *multi) return multi->pipelining_server_bl; } void Curl_multi_process_pending_handles(struct Curl_multi *multi) { - struct SessionHandle *data; + struct curl_llist_element *e; - data=multi->easyp; - while(data) { + for(e = multi->pending->head; e; e = e->next) { + struct SessionHandle *data = e->ptr; if(data->mstate == CURLM_STATE_CONNECT_PEND) { multistate(data, CURLM_STATE_CONNECT); + /* Remove this node from the list */ + Curl_llist_remove(multi->pending, e, NULL); /* Make sure that the handle will be processed soonish. */ - Curl_expire(data, 1); + Curl_expire_latest(data, 1); } - data = data->next; /* operate on next handle */ } } #ifdef DEBUGBUILD void Curl_multi_dump(const struct Curl_multi *multi_handle) diff --git a/lib/multihandle.h b/lib/multihandle.h index 552aa93..1a4b1d9 100644 --- a/lib/multihandle.h +++ b/lib/multihandle.h @@ -5,11 +5,11 @@ * Project ___| | | | _ \| | * / __| | | | |_) | | * | (__| |_| | _ <| |___ * \___|\___/|_| \_\_____| * - * Copyright (C) 1998 - 2013, Daniel Stenberg, <dan...@haxx.se>, et al. + * Copyright (C) 1998 - 2014, Daniel Stenberg, <dan...@haxx.se>, et al. * * This software is licensed as described in the file COPYING, which * you should have received as part of this distribution. The terms * are also available at http://curl.haxx.se/docs/copyright.html. * @@ -73,10 +73,13 @@ struct Curl_multi { int num_alive; /* amount of easy handles that are added but have not yet reached COMPLETE state */ struct curl_llist *msglist; /* a list of messages from completed transfers */ + struct curl_llist *pending; /* SessionHandles that are in the + CURLM_STATE_CONNECT_PEND state */ + /* callback function and user data pointer for the *socket() API */ curl_socket_callback socket_cb; void *socket_userp; /* Hostname cache */ -- 2.1.0
------------------------------------------------------------------- List admin: http://cool.haxx.se/list/listinfo/curl-library Etiquette: http://curl.haxx.se/mail/etiquette.html
