On Sun, Apr 08, 2012 at 06:43:19PM +0300, Yonit Halperin wrote: > Previously, the mjpeg quality was always 70. The frame rate was tuned > according to the frames' congestion in the pipe. > This patch sets the mjpeg quality and frame rate according > to the compressed size of the frames and the currently available bit > rate. > The compression size is estimated for different jpeg qualities, > and the bit rate is evaluated using qos queries (see red_channel). > The bit rate and compression size are monitored for major changes, and > when they occur, the mjpeg settings are re-evaluated. > In addition, the settings are fine-tuned correspondingly to the frames > pipe congestion. > > Signed-off-by: Yonit Halperin <yhalp...@redhat.com> > --- > server/red_worker.c | 385 > ++++++++++++++++++++++++++++++++++++++++++++++----- > 1 files changed, 347 insertions(+), 38 deletions(-)
Sorry, but NACK because of the red_worker.c size increase and $ wc -l red_worker.c 11341 red_worker.c I'm under the impression that the 'jpeg' struct and the methods manipulating it could be moved to a separate file. Christophe > > diff --git a/server/red_worker.c b/server/red_worker.c > index a9942cf..92e1197 100644 > --- a/server/red_worker.c > +++ b/server/red_worker.c > @@ -114,7 +114,8 @@ > #define RED_STREAM_MIN_SIZE (96 * 96) > > #define FPS_TEST_INTERVAL 1 > -#define MAX_FPS 30 > +#define STREAM_MAX_FPS 30 > +#define STREAM_MIN_FPS 1 > > //best bit rate per pixel base on 13000000 bps for frame size 720x576 pixels > and 25 fps > #define BEST_BIT_RATE_PER_PIXEL 38 > @@ -396,6 +397,13 @@ struct Stream { > int bit_rate; > }; > > +#define STREAM_JPEG_QUALITY_SAMPLE_NUM 4 > +static const int stream_jpeg_quality_samples[STREAM_JPEG_QUALITY_SAMPLE_NUM] > = {15, 25, 50, 70}; > + > +#define STREAM_FRAME_SIZE_CHANGE_TH 1.5 > +#define STREAM_BIT_RATE_CHANGE_TH 1.25 > +#define STREAM_AVERAGE_SIZE_WINDOW 3 > + > typedef struct StreamAgent { > QRegion vis_region; > PipeItem create_item; > @@ -403,6 +411,34 @@ typedef struct StreamAgent { > Stream *stream; > uint64_t last_send_time; > > + /* > + Adjusting the stream jpeg quality and frame rate (fps): > + When during_sampling=TRUE, we compress different frames with different > + jpeg quality. By using (1) the resulting compression ratio, (2) the > current > + channel bandwidth, and (3) the existance of other streams, > + we evaulate the max frame frequency for the stream with the given > quality, > + and we choose the highest quality that will allow a reasonable frame > rate. > + during_sampling is set for new streams and also when the bandwidth > and/or > + average frame size significantly change. > + */ > + struct { > + int quality_id; > + uint64_t quality_sample_size[STREAM_JPEG_QUALITY_SAMPLE_NUM]; > + int during_sampling; > + /* low limit for the the current sampling */ > + int min_sample_quality_id; > + int min_sample_quality_fps; // min fps for the given quality > + /* tracking the best sampled fps so far */ > + int max_sampled_fps; > + int max_sampled_fps_quality_id; > + int byte_rate; > + /* tracking the average frame size with the current jpeg quality */ > + uint64_t size_sum; > + int size_summed_count; > + uint64_t recent_size_sum; > + int recent_size_summed_count; > + } jpeg; > + > int frames; > int drops; > int fps; > @@ -938,6 +974,7 @@ typedef struct RedWorker { > Ring streams; > ItemTrace items_trace[NUM_TRACE_ITEMS]; > uint32_t next_item_trace; > + uint64_t streams_size_total; > > QuicData quic_data; > QuicContext *quic; > @@ -2512,6 +2549,7 @@ static void red_attach_stream(RedWorker *worker, > Drawable *drawable, Stream *str > region_clone(&agent->vis_region, &drawable->tree_item.base.rgn); > push_stream_clip_by_drawable(dcc, agent, drawable); > } > + agent->frames++; > } > } > > @@ -2522,6 +2560,7 @@ static void red_stop_stream(RedWorker *worker, Stream > *stream) > > spice_assert(ring_item_is_linked(&stream->link)); > spice_assert(!stream->current); > + spice_debug("id %ld", stream - worker->streams_buf); > WORKER_FOREACH_DCC(worker, item, dcc) { > StreamAgent *stream_agent; > stream_agent = &dcc->stream_agents[stream - worker->streams_buf]; > @@ -2530,6 +2569,7 @@ static void red_stop_stream(RedWorker *worker, Stream > *stream) > stream->refs++; > red_channel_client_pipe_add(&dcc->common.base, > &stream_agent->destroy_item); > } > + worker->streams_size_total -= stream->width * stream->height; > ring_remove(&stream->link); > red_release_stream(worker, stream); > } > @@ -2741,11 +2781,10 @@ static int get_minimal_bit_rate(RedWorker *worker, > int width, int height) > return ret; > } > > -static void red_display_create_stream(DisplayChannelClient *dcc, Stream > *stream) > +static void red_stream_agent_init(DisplayChannelClient *dcc, Stream *stream) > { > StreamAgent *agent = &dcc->stream_agents[stream - > dcc->common.worker->streams_buf]; > > - stream->refs++; > spice_assert(region_is_empty(&agent->vis_region)); > if (stream->current) { > agent->frames = 1; > @@ -2754,8 +2793,36 @@ static void > red_display_create_stream(DisplayChannelClient *dcc, Stream *stream) > agent->frames = 0; > } > agent->drops = 0; > - agent->fps = MAX_FPS; > + agent->fps = STREAM_MAX_FPS; > reset_rate(dcc, agent); > + agent->jpeg.quality_id = STREAM_JPEG_QUALITY_SAMPLE_NUM / 2; > + memset(agent->jpeg.quality_sample_size, 0, > + sizeof(agent->jpeg.quality_sample_size[0]) * > STREAM_JPEG_QUALITY_SAMPLE_NUM); > + agent->jpeg.byte_rate = 0; > + agent->jpeg.max_sampled_fps = 0; > + agent->jpeg.max_sampled_fps_quality_id = 0; > + agent->jpeg.min_sample_quality_id = 0; > + agent->jpeg.min_sample_quality_fps = 0; > + agent->jpeg.size_sum = 0; > + agent->jpeg.size_summed_count = 0; > + agent->jpeg.recent_size_sum = 0; > + agent->jpeg.recent_size_summed_count = 0; > + agent->jpeg.during_sampling = FALSE; > +} > + > +static void red_display_create_stream(DisplayChannelClient *dcc, Stream > *stream) > +{ > + StreamAgent *agent = &dcc->stream_agents[stream - > dcc->common.worker->streams_buf]; > + > + stream->refs++; > + > + spice_debug("id %ld %dx%d dest (%d,%d), (%d, %d)", > + stream - dcc->common.worker->streams_buf, > + stream->width, stream->height, > + stream->dest_area.left, stream->dest_area.top, > + stream->dest_area.right, stream->dest_area.bottom); > + red_stream_agent_init(dcc, stream); > + > red_channel_client_pipe_add(&dcc->common.base, &agent->create_item); > } > > @@ -2794,6 +2861,7 @@ static void red_create_stream(RedWorker *worker, > Drawable *drawable) > SpiceBitmap *bitmap = > &drawable->red_drawable->u.copy.src_bitmap->u.bitmap; > stream->top_down = !!(bitmap->flags & SPICE_BITMAP_FLAGS_TOP_DOWN); > drawable->stream = stream; > + worker->streams_size_total += stream->width * stream->height; > > WORKER_FOREACH_DCC(worker, dcc_ring_item, dcc) { > red_display_create_stream(dcc, stream); > @@ -2945,16 +3013,9 @@ static void reset_rate(DisplayChannelClient *dcc, > StreamAgent *stream_agent) > /* MJpeg has no rate limiting anyway, so do nothing */ > } > > -static int display_channel_client_is_low_bandwidth(DisplayChannelClient *dcc) > -{ > - return main_channel_client_is_low_bandwidth( > - > red_client_get_main(red_channel_client_get_client(&dcc->common.base))); > -} > - > static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream) > { > DrawablePipeItem *dpi; > - DisplayChannelClient *dcc; > int index; > StreamAgent *agent; > RingItem *ring_item; > @@ -2967,35 +3028,10 @@ static inline void pre_stream_item_swap(RedWorker > *worker, Stream *stream) > > index = stream - worker->streams_buf; > DRAWABLE_FOREACH_DPI(stream->current, ring_item, dpi) { > - dcc = dpi->dcc; > - if (!display_channel_client_is_low_bandwidth(dcc)) { > - continue; > - } > - agent = &dcc->stream_agents[index]; > - > + agent = &dpi->dcc->stream_agents[index]; > if (pipe_item_is_linked(&dpi->dpi_pipe_item)) { > ++agent->drops; > } > - > - if (agent->frames / agent->fps < FPS_TEST_INTERVAL) { > - agent->frames++; > - return; > - } > - > - double drop_factor = ((double)agent->frames - (double)agent->drops) / > - (double)agent->frames; > - > - if (drop_factor == 1) { > - if (agent->fps < MAX_FPS) { > - agent->fps++; > - } > - } else if (drop_factor < 0.9) { > - if (agent->fps > 1) { > - agent->fps--; > - } > - } > - agent->frames = 1; > - agent->drops = 0; > } > } > > @@ -8078,6 +8114,274 @@ static int encode_frame (RedWorker *worker, const > SpiceRect *src, > return TRUE; > } > > +static inline void red_stream_agent_jpeg_quality_set(StreamAgent *agent, int > quality_id) > +{ > + if (!agent->jpeg.during_sampling) { > + agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = 0; > + } > + agent->jpeg.size_sum = 0; > + agent->jpeg.size_summed_count = 0; > + agent->jpeg.recent_size_sum = 0; > + agent->jpeg.recent_size_summed_count = 0; > + agent->jpeg.quality_id = quality_id; > +} > + > +/* > + Adjust the stream's jpeg quality and frame rate. > + We evaluate the compression ratio of different jpeg qualities; > + We compress successive frames with different qualities, > + and then we estimate the stream frame rate with the current jpeg quality > + and availalbe bit rate. > +*/ > +static inline void red_stream_do_quality_size_sampling(DisplayChannelClient > *dcc, StreamAgent *agent) > +{ > + int fps; > + int stream_id = agent - dcc->stream_agents; > + > + spice_assert(agent->jpeg.during_sampling); > + if (agent->jpeg.quality_sample_size[agent->jpeg.quality_id] == 0) { > + return; > + } > + > + fps = agent->jpeg.byte_rate / > agent->jpeg.quality_sample_size[agent->jpeg.quality_id]; > + spice_debug("stream %d: jpeg %d: %.2f (KB) fps %d", > + stream_id, > + stream_jpeg_quality_samples[agent->jpeg.quality_id], > + > agent->jpeg.quality_sample_size[agent->jpeg.quality_id]/1000.0, > + fps); > + > + if (fps > agent->jpeg.max_sampled_fps || > + (fps == agent->jpeg.max_sampled_fps && > + agent->jpeg.quality_id > agent->jpeg.max_sampled_fps_quality_id)) { > + agent->jpeg.max_sampled_fps = fps; > + agent->jpeg.max_sampled_fps_quality_id = agent->jpeg.quality_id; > + } > + > + // assuming monotonicity > + if (fps > 5 && fps >= 0.75 * agent->jpeg.min_sample_quality_fps) { > + if (agent->jpeg.quality_id + 1 == STREAM_JPEG_QUALITY_SAMPLE_NUM || > + agent->jpeg.quality_sample_size[agent->jpeg.quality_id + 1] != > 0) { > + /* best quality has been reached, or the next better quality was > + * already sampled and didn't pass the fps threshold */ > + goto complete_sample; > + } else { > + agent->jpeg.quality_id++; > + } > + } else { > + if (agent->jpeg.quality_id == 0 || > + agent->jpeg.quality_id <= agent->jpeg.min_sample_quality_id) { > + goto complete_sample; > + } else if (agent->jpeg.quality_sample_size[agent->jpeg.quality_id - > 1] != 0) { > + agent->jpeg.quality_id--; > + goto complete_sample; > + } else { > + agent->jpeg.quality_id--; > + } > + } > + return; > +complete_sample: > + agent->jpeg.quality_id = MAX(agent->jpeg.quality_id, > + agent->jpeg.max_sampled_fps_quality_id); > + agent->fps = agent->jpeg.byte_rate / > + agent->jpeg.quality_sample_size[agent->jpeg.quality_id]; > + if (agent->jpeg.quality_id == agent->jpeg.min_sample_quality_id) { > + agent->fps = MAX(agent->fps, agent->jpeg.min_sample_quality_fps); > + } > + agent->fps = MIN(STREAM_MAX_FPS, agent->fps); > + agent->fps = MAX(STREAM_MIN_FPS, agent->fps); > + agent->jpeg.during_sampling = FALSE; > + agent->frames = 1; > + agent->drops = 0; > + agent->jpeg.max_sampled_fps = 0; > + agent->jpeg.max_sampled_fps_quality_id = 0; > + agent->jpeg.min_sample_quality_id = 0; > + agent->jpeg.min_sample_quality_fps = 0; > + agent->jpeg.size_sum = 0; > + agent->jpeg.size_summed_count = 0; > + agent->jpeg.recent_size_sum = > agent->jpeg.quality_sample_size[agent->jpeg.quality_id]; > + agent->jpeg.recent_size_summed_count = 1; > + memset(agent->jpeg.quality_sample_size, 0, > + sizeof(agent->jpeg.quality_sample_size[0]) * > STREAM_JPEG_QUALITY_SAMPLE_NUM); > + spice_debug("STREAM QUALITY SAMPLE END %d: quality %d fps %d", > + stream_id, > stream_jpeg_quality_samples[agent->jpeg.quality_id], agent->fps); > +} > + > +/* > + Fine tuning of the stream's frame rate and quality using > + the frames congestion in the pipe. > +*/ > +static inline void red_stream_update_quality_by_drops(DisplayChannelClient > *dcc, > + StreamAgent *agent) > +{ > + int stream_id = agent - dcc->stream_agents; > + > + if (agent->drops) { > + if (agent->frames / agent->fps >= FPS_TEST_INTERVAL) { > + double drop_factor = ((double)agent->frames - > (double)agent->drops) / > + (double)agent->frames; > + if (drop_factor <= 0.9) { > + if (agent->fps <= 10 && agent->jpeg.quality_id > 0 ) { > + red_stream_agent_jpeg_quality_set(agent, > agent->jpeg.quality_id - 1); > + spice_debug("stream %d quality--: jpeg %d fps %d", > stream_id, > + > stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + } else { > + agent->fps--; > + agent->fps = MAX(STREAM_MIN_FPS, agent->fps); > + spice_debug("stream %d fps--: jpeg %d fps %d", stream_id, > + > stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + } > + } > + agent->frames = 1; > + agent->drops = 0; > + } > + } else { > + if (agent->frames / agent->fps >= FPS_TEST_INTERVAL) { > + if (agent->fps >= 15 && agent->jpeg.quality_id < > STREAM_JPEG_QUALITY_SAMPLE_NUM - 1) { > + /* being more strict when we want to increase quality */ > + if (agent->frames / agent->fps >= 2 * FPS_TEST_INTERVAL) { > + agent->jpeg.min_sample_quality_id = > agent->jpeg.quality_id; > + agent->jpeg.min_sample_quality_fps = agent->fps; > + agent->jpeg.during_sampling = TRUE; > + red_stream_do_quality_size_sampling(dcc, agent); > + spice_debug("stream %d quality resampling: jpeg %d fps > %d", stream_id, > + > stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + agent->frames = 1; > + agent->drops = 0; > + } else { > + agent->fps++; > + agent->fps = MIN(STREAM_MAX_FPS, agent->fps); > + spice_debug("stream %d fps++: jpeg %d fps %d", stream_id, > + > stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + } > + } else { > + agent->fps++; > + agent->fps = MIN(STREAM_MAX_FPS, agent->fps); > + spice_debug("stream %d fps++: jpeg %d fps %d", stream_id, > + > stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + agent->frames = 1; > + agent->drops = 0; > + } > + } > + } > +} > + > +/* > + Monitor changes in the available byte rate for the stream, > + and/or in the stream's compressed frames size. If the changes > + pass a predefined threshold, we re-evaluate the stream's jpeg > + quality and frame rate. > +*/ > +static void red_stream_update_quality(RedChannelClient *rcc, StreamAgent > *agent) > +{ > + DisplayChannelClient *dcc = RCC_TO_DCC(rcc); > + RedWorker *worker = dcc->common.worker; > + int stream_id = agent->stream - dcc->common.worker->streams_buf; > + int channel_bit_rate; > + int stream_byte_rate; > + float byte_rate_change; > + float size_change = 1; > + double size_avg_old = 0.0; > + double size_avg_new = 0.0; > + > + channel_bit_rate = red_channel_client_get_qos_bit_rate(rcc); > + > + if (channel_bit_rate == 0) { > + spice_assert(agent->jpeg.byte_rate == 0); > + red_stream_update_quality_by_drops(dcc, agent); > + return; > + } > + > + stream_byte_rate = 0.8 * channel_bit_rate * (agent->stream->width * > agent->stream->height) / > + worker->streams_size_total / 8; > + if (agent->jpeg.byte_rate == 0) { // new stream > + agent->jpeg.byte_rate = stream_byte_rate; > + agent->jpeg.during_sampling = TRUE; > + agent->jpeg.min_sample_quality_id = 0; > + agent->jpeg.min_sample_quality_fps = 0; > + } > + > + if (agent->jpeg.during_sampling) { > + red_stream_do_quality_size_sampling(dcc, agent); > + return; > + } > + spice_assert(agent->jpeg.quality_sample_size[agent->jpeg.quality_id]); > + byte_rate_change = (stream_byte_rate + 0.0) / agent->jpeg.byte_rate; > + > + agent->jpeg.recent_size_sum += > agent->jpeg.quality_sample_size[agent->jpeg.quality_id]; > + agent->jpeg.recent_size_summed_count++; > + size_avg_new = (agent->jpeg.recent_size_sum + 0.0) / > agent->jpeg.recent_size_summed_count; > + if (agent->jpeg.recent_size_summed_count >= STREAM_AVERAGE_SIZE_WINDOW && > + agent->jpeg.size_summed_count > 0) { > + size_avg_old = (agent->jpeg.size_sum + 0.0) / > agent->jpeg.size_summed_count; > + size_change = size_avg_new / size_avg_old; > + } > + > + if (byte_rate_change > STREAM_BIT_RATE_CHANGE_TH) { > + spice_debug("stream %d BYTE RATE CHANGE >>: %.2f (%d-->%d > prev-quality %d prev-fps %d", > + stream_id, byte_rate_change, agent->jpeg.byte_rate, > stream_byte_rate, > + stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + agent->jpeg.during_sampling = TRUE; > + /* byte rate has improved --> don't allow stream to deteriorate */ > + agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id; > + agent->jpeg.min_sample_quality_fps = agent->fps; > + } else if (1.0 / byte_rate_change > STREAM_BIT_RATE_CHANGE_TH) { > + spice_debug("stream %d BYTE RATE CHANGE <<: %.2f (%d-->%d) > prev-quality %d prev-fps %d", > + stream_id, byte_rate_change, agent->jpeg.byte_rate, > stream_byte_rate, > + stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + if (agent->fps * size_avg_new > stream_byte_rate) { > + agent->jpeg.during_sampling = TRUE; > + agent->jpeg.min_sample_quality_id = 0; > + agent->jpeg.min_sample_quality_fps = 0; > + } else { > + spice_debug("stream byte rate is not limiting the current > setting"); > + } > + } > + > + if (size_change > STREAM_FRAME_SIZE_CHANGE_TH) { > + spice_debug("stream %d SIZE CHANGE >>: %.2f (%.2f-->%.2f) > prev-quality %d prev-fps %d", > + stream_id, size_change, size_avg_old, size_avg_new, > + stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + if (agent->fps * size_avg_new > stream_byte_rate) { > + agent->jpeg.during_sampling = TRUE; > + agent->jpeg.min_sample_quality_id = 0; > + agent->jpeg.min_sample_quality_fps = 0; > + } else { > + spice_debug("stream frame size is not limiting current setting"); > + } > + } else if (1.0 / size_change > STREAM_FRAME_SIZE_CHANGE_TH) { > + spice_debug("stream %d SIZE CHANGE <<: %.2f (%.2f-->%.2f) > prev-quality %d prev-fps %d", > + stream_id, size_change, size_avg_old, size_avg_new, > + stream_jpeg_quality_samples[agent->jpeg.quality_id], > + agent->fps); > + agent->jpeg.during_sampling = TRUE; > + /* compression ratio has improved --> don't allow stream to > deteriorate */ > + agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id; > + agent->jpeg.min_sample_quality_fps = agent->fps; > + } > + agent->jpeg.byte_rate = stream_byte_rate; > + > + if (agent->jpeg.recent_size_summed_count >= STREAM_AVERAGE_SIZE_WINDOW) { > + agent->jpeg.size_sum += agent->jpeg.recent_size_sum; > + agent->jpeg.size_summed_count += > agent->jpeg.recent_size_summed_count; > + agent->jpeg.recent_size_sum = 0; > + agent->jpeg.recent_size_summed_count = 0; > + } > + if (agent->jpeg.during_sampling) { > + agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = > size_avg_new; > + red_stream_do_quality_size_sampling(dcc, agent); > + } else { > + red_stream_update_quality_by_drops(dcc, agent); > + } > +} > + > static inline int red_marshall_stream_data(RedChannelClient *rcc, > SpiceMarshaller *base_marshaller, Drawable *drawable) > { > @@ -8124,21 +8428,26 @@ static inline int > red_marshall_stream_data(RedChannelClient *rcc, > return TRUE; > } > > + red_stream_update_quality(rcc, agent); > outbuf_size = dcc->send_data.stream_outbuf_size; > if (!mjpeg_encoder_start_frame(stream->mjpeg_encoder, > image->u.bitmap.format, > - 70, > + > stream_jpeg_quality_samples[agent->jpeg.quality_id], > width, height, > &dcc->send_data.stream_outbuf, > &outbuf_size)) { > + spice_debug("start frame failed"); > return FALSE; > } > if (!encode_frame(worker, &drawable->red_drawable->u.copy.src_area, > &image->u.bitmap, stream)) { > + spice_debug("encode frame failed"); > return FALSE; > } > n = mjpeg_encoder_end_frame(stream->mjpeg_encoder); > dcc->send_data.stream_outbuf_size = outbuf_size; > > + agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = n; > + > if (!drawable->sized_stream) { > SpiceMsgDisplayStreamData stream_data; > > -- > 1.7.7.6 > > _______________________________________________ > Spice-devel mailing list > Spice-devel@lists.freedesktop.org > http://lists.freedesktop.org/mailman/listinfo/spice-devel
pgpbjrn4nM35E.pgp
Description: PGP signature
_______________________________________________ Spice-devel mailing list Spice-devel@lists.freedesktop.org http://lists.freedesktop.org/mailman/listinfo/spice-devel