diff --git a/server/red_worker.c b/server/red_worker.c
index a9942cf..92e1197 100644
--- a/server/red_worker.c
+++ b/server/red_worker.c
@@ -114,7 +114,8 @@
#define RED_STREAM_MIN_SIZE (96 * 96)
#define FPS_TEST_INTERVAL 1
-#define MAX_FPS 30
+#define STREAM_MAX_FPS 30
+#define STREAM_MIN_FPS 1
//best bit rate per pixel base on 13000000 bps for frame size 720x576 pixels
and 25 fps
#define BEST_BIT_RATE_PER_PIXEL 38
@@ -396,6 +397,13 @@ struct Stream {
int bit_rate;
};
+#define STREAM_JPEG_QUALITY_SAMPLE_NUM 4
+static const int stream_jpeg_quality_samples[STREAM_JPEG_QUALITY_SAMPLE_NUM] =
{15, 25, 50, 70};
+
+#define STREAM_FRAME_SIZE_CHANGE_TH 1.5
+#define STREAM_BIT_RATE_CHANGE_TH 1.25
+#define STREAM_AVERAGE_SIZE_WINDOW 3
+
typedef struct StreamAgent {
QRegion vis_region;
PipeItem create_item;
@@ -403,6 +411,34 @@ typedef struct StreamAgent {
Stream *stream;
uint64_t last_send_time;
+ /*
+ Adjusting the stream jpeg quality and frame rate (fps):
+ When during_sampling=TRUE, we compress different frames with different
+ jpeg quality. By using (1) the resulting compression ratio, (2) the
current
+ channel bandwidth, and (3) the existance of other streams,
+ we evaulate the max frame frequency for the stream with the given
quality,
+ and we choose the highest quality that will allow a reasonable frame
rate.
+ during_sampling is set for new streams and also when the bandwidth and/or
+ average frame size significantly change.
+ */
+ struct {
+ int quality_id;
+ uint64_t quality_sample_size[STREAM_JPEG_QUALITY_SAMPLE_NUM];
+ int during_sampling;
+ /* low limit for the the current sampling */
+ int min_sample_quality_id;
+ int min_sample_quality_fps; // min fps for the given quality
+ /* tracking the best sampled fps so far */
+ int max_sampled_fps;
+ int max_sampled_fps_quality_id;
+ int byte_rate;
+ /* tracking the average frame size with the current jpeg quality */
+ uint64_t size_sum;
+ int size_summed_count;
+ uint64_t recent_size_sum;
+ int recent_size_summed_count;
+ } jpeg;
+
int frames;
int drops;
int fps;
@@ -938,6 +974,7 @@ typedef struct RedWorker {
Ring streams;
ItemTrace items_trace[NUM_TRACE_ITEMS];
uint32_t next_item_trace;
+ uint64_t streams_size_total;
QuicData quic_data;
QuicContext *quic;
@@ -2512,6 +2549,7 @@ static void red_attach_stream(RedWorker *worker, Drawable
*drawable, Stream *str
region_clone(&agent->vis_region,&drawable->tree_item.base.rgn);
push_stream_clip_by_drawable(dcc, agent, drawable);
}
+ agent->frames++;
}
}
@@ -2522,6 +2560,7 @@ static void red_stop_stream(RedWorker *worker, Stream
*stream)
spice_assert(ring_item_is_linked(&stream->link));
spice_assert(!stream->current);
+ spice_debug("id %ld", stream - worker->streams_buf);
WORKER_FOREACH_DCC(worker, item, dcc) {
StreamAgent *stream_agent;
stream_agent =&dcc->stream_agents[stream - worker->streams_buf];
@@ -2530,6 +2569,7 @@ static void red_stop_stream(RedWorker *worker, Stream
*stream)
stream->refs++;
red_channel_client_pipe_add(&dcc->common.base,&stream_agent->destroy_item);
}
+ worker->streams_size_total -= stream->width * stream->height;
ring_remove(&stream->link);
red_release_stream(worker, stream);
}
@@ -2741,11 +2781,10 @@ static int get_minimal_bit_rate(RedWorker *worker, int
width, int height)
return ret;
}
-static void red_display_create_stream(DisplayChannelClient *dcc, Stream
*stream)
+static void red_stream_agent_init(DisplayChannelClient *dcc, Stream *stream)
{
StreamAgent *agent =&dcc->stream_agents[stream -
dcc->common.worker->streams_buf];
- stream->refs++;
spice_assert(region_is_empty(&agent->vis_region));
if (stream->current) {
agent->frames = 1;
@@ -2754,8 +2793,36 @@ static void
red_display_create_stream(DisplayChannelClient *dcc, Stream *stream)
agent->frames = 0;
}
agent->drops = 0;
- agent->fps = MAX_FPS;
+ agent->fps = STREAM_MAX_FPS;
reset_rate(dcc, agent);
+ agent->jpeg.quality_id = STREAM_JPEG_QUALITY_SAMPLE_NUM / 2;
+ memset(agent->jpeg.quality_sample_size, 0,
+ sizeof(agent->jpeg.quality_sample_size[0]) *
STREAM_JPEG_QUALITY_SAMPLE_NUM);
+ agent->jpeg.byte_rate = 0;
+ agent->jpeg.max_sampled_fps = 0;
+ agent->jpeg.max_sampled_fps_quality_id = 0;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ agent->jpeg.size_sum = 0;
+ agent->jpeg.size_summed_count = 0;
+ agent->jpeg.recent_size_sum = 0;
+ agent->jpeg.recent_size_summed_count = 0;
+ agent->jpeg.during_sampling = FALSE;
+}
+
+static void red_display_create_stream(DisplayChannelClient *dcc, Stream
*stream)
+{
+ StreamAgent *agent =&dcc->stream_agents[stream -
dcc->common.worker->streams_buf];
+
+ stream->refs++;
+
+ spice_debug("id %ld %dx%d dest (%d,%d), (%d, %d)",
+ stream - dcc->common.worker->streams_buf,
+ stream->width, stream->height,
+ stream->dest_area.left, stream->dest_area.top,
+ stream->dest_area.right, stream->dest_area.bottom);
+ red_stream_agent_init(dcc, stream);
+
red_channel_client_pipe_add(&dcc->common.base,&agent->create_item);
}
@@ -2794,6 +2861,7 @@ static void red_create_stream(RedWorker *worker, Drawable
*drawable)
SpiceBitmap *bitmap =&drawable->red_drawable->u.copy.src_bitmap->u.bitmap;
stream->top_down = !!(bitmap->flags& SPICE_BITMAP_FLAGS_TOP_DOWN);
drawable->stream = stream;
+ worker->streams_size_total += stream->width * stream->height;
WORKER_FOREACH_DCC(worker, dcc_ring_item, dcc) {
red_display_create_stream(dcc, stream);
@@ -2945,16 +3013,9 @@ static void reset_rate(DisplayChannelClient *dcc,
StreamAgent *stream_agent)
/* MJpeg has no rate limiting anyway, so do nothing */
}
-static int display_channel_client_is_low_bandwidth(DisplayChannelClient *dcc)
-{
- return main_channel_client_is_low_bandwidth(
- red_client_get_main(red_channel_client_get_client(&dcc->common.base)));
-}
-
static inline void pre_stream_item_swap(RedWorker *worker, Stream *stream)
{
DrawablePipeItem *dpi;
- DisplayChannelClient *dcc;
int index;
StreamAgent *agent;
RingItem *ring_item;
@@ -2967,35 +3028,10 @@ static inline void pre_stream_item_swap(RedWorker
*worker, Stream *stream)
index = stream - worker->streams_buf;
DRAWABLE_FOREACH_DPI(stream->current, ring_item, dpi) {
- dcc = dpi->dcc;
- if (!display_channel_client_is_low_bandwidth(dcc)) {
- continue;
- }
- agent =&dcc->stream_agents[index];
-
+ agent =&dpi->dcc->stream_agents[index];
if (pipe_item_is_linked(&dpi->dpi_pipe_item)) {
++agent->drops;
}
-
- if (agent->frames / agent->fps< FPS_TEST_INTERVAL) {
- agent->frames++;
- return;
- }
-
- double drop_factor = ((double)agent->frames - (double)agent->drops) /
- (double)agent->frames;
-
- if (drop_factor == 1) {
- if (agent->fps< MAX_FPS) {
- agent->fps++;
- }
- } else if (drop_factor< 0.9) {
- if (agent->fps> 1) {
- agent->fps--;
- }
- }
- agent->frames = 1;
- agent->drops = 0;
}
}
@@ -8078,6 +8114,274 @@ static int encode_frame (RedWorker *worker, const
SpiceRect *src,
return TRUE;
}
+static inline void red_stream_agent_jpeg_quality_set(StreamAgent *agent, int
quality_id)
+{
+ if (!agent->jpeg.during_sampling) {
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = 0;
+ }
+ agent->jpeg.size_sum = 0;
+ agent->jpeg.size_summed_count = 0;
+ agent->jpeg.recent_size_sum = 0;
+ agent->jpeg.recent_size_summed_count = 0;
+ agent->jpeg.quality_id = quality_id;
+}
+
+/*
+ Adjust the stream's jpeg quality and frame rate.
+ We evaluate the compression ratio of different jpeg qualities;
+ We compress successive frames with different qualities,
+ and then we estimate the stream frame rate with the current jpeg quality
+ and availalbe bit rate.
+*/
+static inline void red_stream_do_quality_size_sampling(DisplayChannelClient
*dcc, StreamAgent *agent)
+{
+ int fps;
+ int stream_id = agent - dcc->stream_agents;
+
+ spice_assert(agent->jpeg.during_sampling);
+ if (agent->jpeg.quality_sample_size[agent->jpeg.quality_id] == 0) {
+ return;
+ }
+
+ fps = agent->jpeg.byte_rate /
agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ spice_debug("stream %d: jpeg %d: %.2f (KB) fps %d",
+ stream_id,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id]/1000.0,
+ fps);
+
+ if (fps> agent->jpeg.max_sampled_fps ||
+ (fps == agent->jpeg.max_sampled_fps&&
+ agent->jpeg.quality_id> agent->jpeg.max_sampled_fps_quality_id)) {
+ agent->jpeg.max_sampled_fps = fps;
+ agent->jpeg.max_sampled_fps_quality_id = agent->jpeg.quality_id;
+ }
+
+ // assuming monotonicity
+ if (fps> 5&& fps>= 0.75 * agent->jpeg.min_sample_quality_fps) {
+ if (agent->jpeg.quality_id + 1 == STREAM_JPEG_QUALITY_SAMPLE_NUM ||
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id + 1] != 0) {
+ /* best quality has been reached, or the next better quality was
+ * already sampled and didn't pass the fps threshold */
+ goto complete_sample;
+ } else {
+ agent->jpeg.quality_id++;
+ }
+ } else {
+ if (agent->jpeg.quality_id == 0 ||
+ agent->jpeg.quality_id<= agent->jpeg.min_sample_quality_id) {
+ goto complete_sample;
+ } else if (agent->jpeg.quality_sample_size[agent->jpeg.quality_id - 1]
!= 0) {
+ agent->jpeg.quality_id--;
+ goto complete_sample;
+ } else {
+ agent->jpeg.quality_id--;
+ }
+ }
+ return;
+complete_sample:
+ agent->jpeg.quality_id = MAX(agent->jpeg.quality_id,
+ agent->jpeg.max_sampled_fps_quality_id);
+ agent->fps = agent->jpeg.byte_rate /
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ if (agent->jpeg.quality_id == agent->jpeg.min_sample_quality_id) {
+ agent->fps = MAX(agent->fps, agent->jpeg.min_sample_quality_fps);
+ }
+ agent->fps = MIN(STREAM_MAX_FPS, agent->fps);
+ agent->fps = MAX(STREAM_MIN_FPS, agent->fps);
+ agent->jpeg.during_sampling = FALSE;
+ agent->frames = 1;
+ agent->drops = 0;
+ agent->jpeg.max_sampled_fps = 0;
+ agent->jpeg.max_sampled_fps_quality_id = 0;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ agent->jpeg.size_sum = 0;
+ agent->jpeg.size_summed_count = 0;
+ agent->jpeg.recent_size_sum =
agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ agent->jpeg.recent_size_summed_count = 1;
+ memset(agent->jpeg.quality_sample_size, 0,
+ sizeof(agent->jpeg.quality_sample_size[0]) *
STREAM_JPEG_QUALITY_SAMPLE_NUM);
+ spice_debug("STREAM QUALITY SAMPLE END %d: quality %d fps %d",
+ stream_id, stream_jpeg_quality_samples[agent->jpeg.quality_id],
agent->fps);
+}
+
+/*
+ Fine tuning of the stream's frame rate and quality using
+ the frames congestion in the pipe.
+*/
+static inline void red_stream_update_quality_by_drops(DisplayChannelClient
*dcc,
+ StreamAgent *agent)
+{
+ int stream_id = agent - dcc->stream_agents;
+
+ if (agent->drops) {
+ if (agent->frames / agent->fps>= FPS_TEST_INTERVAL) {
+ double drop_factor = ((double)agent->frames -
(double)agent->drops) /
+ (double)agent->frames;
+ if (drop_factor<= 0.9) {
+ if (agent->fps<= 10&& agent->jpeg.quality_id> 0 ) {
+ red_stream_agent_jpeg_quality_set(agent,
agent->jpeg.quality_id - 1);
+ spice_debug("stream %d quality--: jpeg %d fps %d",
stream_id,
+
stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ } else {
+ agent->fps--;
+ agent->fps = MAX(STREAM_MIN_FPS, agent->fps);
+ spice_debug("stream %d fps--: jpeg %d fps %d", stream_id,
+
stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ }
+ }
+ agent->frames = 1;
+ agent->drops = 0;
+ }
+ } else {
+ if (agent->frames / agent->fps>= FPS_TEST_INTERVAL) {
+ if (agent->fps>= 15&& agent->jpeg.quality_id<
STREAM_JPEG_QUALITY_SAMPLE_NUM - 1) {
+ /* being more strict when we want to increase quality */
+ if (agent->frames / agent->fps>= 2 * FPS_TEST_INTERVAL) {
+ agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id;
+ agent->jpeg.min_sample_quality_fps = agent->fps;
+ agent->jpeg.during_sampling = TRUE;
+ red_stream_do_quality_size_sampling(dcc, agent);
+ spice_debug("stream %d quality resampling: jpeg %d fps
%d", stream_id,
+
stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->frames = 1;
+ agent->drops = 0;
+ } else {
+ agent->fps++;
+ agent->fps = MIN(STREAM_MAX_FPS, agent->fps);
+ spice_debug("stream %d fps++: jpeg %d fps %d", stream_id,
+
stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ }
+ } else {
+ agent->fps++;
+ agent->fps = MIN(STREAM_MAX_FPS, agent->fps);
+ spice_debug("stream %d fps++: jpeg %d fps %d", stream_id,
+
stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->frames = 1;
+ agent->drops = 0;
+ }
+ }
+ }
+}
+
+/*
+ Monitor changes in the available byte rate for the stream,
+ and/or in the stream's compressed frames size. If the changes
+ pass a predefined threshold, we re-evaluate the stream's jpeg
+ quality and frame rate.
+*/
+static void red_stream_update_quality(RedChannelClient *rcc, StreamAgent
*agent)
+{
+ DisplayChannelClient *dcc = RCC_TO_DCC(rcc);
+ RedWorker *worker = dcc->common.worker;
+ int stream_id = agent->stream - dcc->common.worker->streams_buf;
+ int channel_bit_rate;
+ int stream_byte_rate;
+ float byte_rate_change;
+ float size_change = 1;
+ double size_avg_old = 0.0;
+ double size_avg_new = 0.0;
+
+ channel_bit_rate = red_channel_client_get_qos_bit_rate(rcc);
+
+ if (channel_bit_rate == 0) {
+ spice_assert(agent->jpeg.byte_rate == 0);
+ red_stream_update_quality_by_drops(dcc, agent);
+ return;
+ }
+
+ stream_byte_rate = 0.8 * channel_bit_rate * (agent->stream->width *
agent->stream->height) /
+ worker->streams_size_total / 8;
+ if (agent->jpeg.byte_rate == 0) { // new stream
+ agent->jpeg.byte_rate = stream_byte_rate;
+ agent->jpeg.during_sampling = TRUE;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ }
+
+ if (agent->jpeg.during_sampling) {
+ red_stream_do_quality_size_sampling(dcc, agent);
+ return;
+ }
+ spice_assert(agent->jpeg.quality_sample_size[agent->jpeg.quality_id]);
+ byte_rate_change = (stream_byte_rate + 0.0) / agent->jpeg.byte_rate;
+
+ agent->jpeg.recent_size_sum +=
agent->jpeg.quality_sample_size[agent->jpeg.quality_id];
+ agent->jpeg.recent_size_summed_count++;
+ size_avg_new = (agent->jpeg.recent_size_sum + 0.0) /
agent->jpeg.recent_size_summed_count;
+ if (agent->jpeg.recent_size_summed_count>= STREAM_AVERAGE_SIZE_WINDOW&&
+ agent->jpeg.size_summed_count> 0) {
+ size_avg_old = (agent->jpeg.size_sum + 0.0) /
agent->jpeg.size_summed_count;
+ size_change = size_avg_new / size_avg_old;
+ }
+
+ if (byte_rate_change> STREAM_BIT_RATE_CHANGE_TH) {
+ spice_debug("stream %d BYTE RATE CHANGE>>: %.2f (%d-->%d prev-quality %d
prev-fps %d",
+ stream_id, byte_rate_change, agent->jpeg.byte_rate,
stream_byte_rate,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->jpeg.during_sampling = TRUE;
+ /* byte rate has improved --> don't allow stream to deteriorate */
+ agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id;
+ agent->jpeg.min_sample_quality_fps = agent->fps;
+ } else if (1.0 / byte_rate_change> STREAM_BIT_RATE_CHANGE_TH) {
+ spice_debug("stream %d BYTE RATE CHANGE<<: %.2f (%d-->%d) prev-quality %d
prev-fps %d",
+ stream_id, byte_rate_change, agent->jpeg.byte_rate,
stream_byte_rate,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ if (agent->fps * size_avg_new> stream_byte_rate) {
+ agent->jpeg.during_sampling = TRUE;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ } else {
+ spice_debug("stream byte rate is not limiting the current
setting");
+ }
+ }
+
+ if (size_change> STREAM_FRAME_SIZE_CHANGE_TH) {
+ spice_debug("stream %d SIZE CHANGE>>: %.2f (%.2f-->%.2f) prev-quality %d
prev-fps %d",
+ stream_id, size_change, size_avg_old, size_avg_new,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ if (agent->fps * size_avg_new> stream_byte_rate) {
+ agent->jpeg.during_sampling = TRUE;
+ agent->jpeg.min_sample_quality_id = 0;
+ agent->jpeg.min_sample_quality_fps = 0;
+ } else {
+ spice_debug("stream frame size is not limiting current setting");
+ }
+ } else if (1.0 / size_change> STREAM_FRAME_SIZE_CHANGE_TH) {
+ spice_debug("stream %d SIZE CHANGE<<: %.2f (%.2f-->%.2f) prev-quality %d
prev-fps %d",
+ stream_id, size_change, size_avg_old, size_avg_new,
+ stream_jpeg_quality_samples[agent->jpeg.quality_id],
+ agent->fps);
+ agent->jpeg.during_sampling = TRUE;
+ /* compression ratio has improved --> don't allow stream to
deteriorate */
+ agent->jpeg.min_sample_quality_id = agent->jpeg.quality_id;
+ agent->jpeg.min_sample_quality_fps = agent->fps;
+ }
+ agent->jpeg.byte_rate = stream_byte_rate;
+
+ if (agent->jpeg.recent_size_summed_count>= STREAM_AVERAGE_SIZE_WINDOW) {
+ agent->jpeg.size_sum += agent->jpeg.recent_size_sum;
+ agent->jpeg.size_summed_count += agent->jpeg.recent_size_summed_count;
+ agent->jpeg.recent_size_sum = 0;
+ agent->jpeg.recent_size_summed_count = 0;
+ }
+ if (agent->jpeg.during_sampling) {
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = size_avg_new;
+ red_stream_do_quality_size_sampling(dcc, agent);
+ } else {
+ red_stream_update_quality_by_drops(dcc, agent);
+ }
+}
+
static inline int red_marshall_stream_data(RedChannelClient *rcc,
SpiceMarshaller *base_marshaller, Drawable *drawable)
{
@@ -8124,21 +8428,26 @@ static inline int
red_marshall_stream_data(RedChannelClient *rcc,
return TRUE;
}
+ red_stream_update_quality(rcc, agent);
outbuf_size = dcc->send_data.stream_outbuf_size;
if (!mjpeg_encoder_start_frame(stream->mjpeg_encoder,
image->u.bitmap.format,
- 70,
+
stream_jpeg_quality_samples[agent->jpeg.quality_id],
width, height,
&dcc->send_data.stream_outbuf,
&outbuf_size)) {
+ spice_debug("start frame failed");
return FALSE;
}
if (!encode_frame(worker,&drawable->red_drawable->u.copy.src_area,
&image->u.bitmap, stream)) {
+ spice_debug("encode frame failed");
return FALSE;
}
n = mjpeg_encoder_end_frame(stream->mjpeg_encoder);
dcc->send_data.stream_outbuf_size = outbuf_size;
+ agent->jpeg.quality_sample_size[agent->jpeg.quality_id] = n;
+
if (!drawable->sized_stream) {
SpiceMsgDisplayStreamData stream_data;
--
1.7.7.6
_______________________________________________
Spice-devel mailing list
Spice-devel@lists.freedesktop.org
http://lists.freedesktop.org/mailman/listinfo/spice-devel
