The GStreamer encoders don't follow the specified bit rate very closely: they can decide to exceed it for ten seconds or more if they consider the scene deserves it. Such long bursts are enough to cause network congestion, resulting in long bouts of dropped frames.
So the GStreamer video encoder now uses a virtual buffer to shape the compressed video output and ensure we don't exceed the target bit rate for any significant length of time, which makes it possible to use higher bit rates overall.
It also keeps track of the encoded frame size so it can gather statistics and call update_client_playback_delay() with accurate information and also annotate the client report debug traces with the corresponding bit rate information.
Signed-off-by: Francois Gouget <fgouget@xxxxxxxxxxxxxxx>
---
The bit rate control code can be split into two parts:
1. Code to turn the raw video into a compressed stream of the given bit rate.
2. Feedback code that adjusts the bit rate based on the network conditions.
In theory the GStreamer encoders implement part 1 for us but, as stated
in the commit message, in practice they are not strict enough (and most
cannot be tweaked in this respect).
It's interesting to note that if the feedback mechanism gets good,
timely information we can pretty much do without this patch: when
GStreamer exceeds the set bit rate the feedback mechanism will notice a
degradation of the network conditions and lower the target bit rate
which will lower the GStreamer's output bit rate (even if it still
exceeds the target bit rate a bit).
However this is suboptimal as it will force the feedback mechanism to
either change the bit rate more frequently (causing a GStreamer pipeline
reinitialization most of the time and degrading compression levels), or
select a lower bit rate so the network capacity is not exceeded even if
GStreamer goes 10 or 20% above the set target.
Also note that despite wanting to avoid exceeding the target bit rate,
part 1 cannot consider frames individually, if only to correctly handle
VP8's large I vs. P frame size discrepancy. Exceeding the target bit
rate for sub-second periods generally does not cause network congestion
hence the selection of a 300ms virtual buffer.
server/gstreamer_encoder.c | 287 +++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 279 insertions(+), 8 deletions(-)
diff --git a/server/gstreamer_encoder.c b/server/gstreamer_encoder.c
index 52b3e03..4089ead 100644
--- a/server/gstreamer_encoder.c
+++ b/server/gstreamer_encoder.c
@@ -39,6 +39,9 @@ typedef struct GstEncoder GstEncoder;
# define DO_ZERO_COPY
#endif
+#define NANO_SECOND (1000000000LL)
+#define MILLI_SECOND (1000LL)
+#define NANO_MS (NANO_SECOND / MILLI_SECOND)
typedef struct {
SpiceBitmapFmt spice_format;
@@ -59,6 +62,11 @@ struct GstVideoBuffer {
gboolean persistent;
};
+typedef struct {
+ uint32_t mm_time;
+ uint32_t size;
+} GstFrameInformation;
+
struct GstEncoder {
VideoEncoder base;
@@ -98,11 +106,71 @@ struct GstEncoder {
/* The frame counter for GStreamer buffers */
uint32_t frame;
+
+ /* ---------- Encoded frame statistics ---------- */
+
+ /* Should be >= than FRAME_STATISTICS_COUNT. This is also used to annotate
+ * the client report debug traces with bit rate information.
+ */
+# define GSTE_HISTORY_SIZE 60
+
+ /* A circular buffer containing the past encoded frames information. */
+ GstFrameInformation history[GSTE_HISTORY_SIZE];
+
+ /* The indices of the oldest and newest frames in the history buffer. */
+ uint32_t history_first;
+ uint32_t history_last;
+
+ /* How many frames to take into account when computing the effective
+ * bit rate, average frame size, etc. This should be large enough so the
+ * I and P frames average out, and short enough for it to reflect the
+ * current situation.
+ */
+# define GSTE_FRAME_STATISTICS_COUNT 21
+
+ /* The index of the oldest frame taken into account for the statistics. */
+ uint32_t stat_first;
+
+ /* Used to compute the average frame size. */
+ uint64_t stat_sum;
+
+ /* Tracks the maximum frame size. */
+ uint32_t stat_maximum;
+
+
+ /* ---------- Encoder bit rate control ----------
+ *
+ * GStreamer encoders don't follow the specified bit rate very
+ * closely. These fields are used to ensure we don't exceed the desired
+ * stream bit rate, regardless of the GStreamer encoder's output.
+ */
+
/* The bit rate target for the outgoing network stream. (bits per second) */
uint64_t bit_rate;
/* The minimum bit rate */
# define GSTE_MIN_BITRATE (128 * 1024)
+
+ /* The bit rate control is performed using a virtual buffer to allow short
+ * term variations: bursts are allowed until the virtual buffer is full.
+ * Then frames are dropped to limit the bit rate. VBUFFER_SIZE defines the
+ * size of the virtual buffer in milliseconds worth of data.
+ */
+# define GSTE_VBUFFER_SIZE 300
+
+ int32_t vbuffer_size;
+ int32_t vbuffer_free;
+
+ /* When dropping frames, this is set to the minimum mm_time of the next
+ * frame to encode. Otherwise set to zero.
+ */
+ uint32_t next_frame;
+
+ /* Defines the minimum allowed fps. */
+# define GSTE_MAX_PERIOD (NANO_SECOND / 3)
+
+ /* How big of a margin to take to cover for latency jitter. */
+# define GSTE_LATENCY_MARGIN 0.1
};
@@ -154,6 +222,13 @@ static uint32_t get_source_fps(GstEncoder *encoder)
encoder->cbs.get_source_fps(encoder->cbs_opaque) : GSTE_DEFAULT_FPS;
}
+static uint32_t get_network_latency(GstEncoder *encoder)
+{
+ /* Assume that the network latency is symmetric */
+ return encoder->cbs.get_roundtrip_ms ?
+ encoder->cbs.get_roundtrip_ms(encoder->cbs_opaque) / 2 : 0;
+}
+
static void reset_pipeline(GstEncoder *encoder)
{
if (!encoder->pipeline) {
@@ -169,6 +244,180 @@ static void reset_pipeline(GstEncoder *encoder)
encoder->pipeline = NULL;
}
+
+/* ---------- Encoded frame statistics ---------- */
+
+static inline uint32_t get_last_frame_mm_time(GstEncoder *encoder)
+{
+ return encoder->history[encoder->history_last].mm_time;
+}
+
+/* Returns the current bit rate based on the last GSTE_FRAME_STATISTICS_COUNT
+ * frames.
+ */
+static uint64_t get_effective_bit_rate(GstEncoder *encoder)
+{
+ uint32_t elapsed = encoder->history[encoder->history_last].mm_time -
+ encoder->history[encoder->stat_first].mm_time;
+ if (encoder->next_frame) {
+ elapsed += encoder->next_frame - get_last_frame_mm_time(encoder);
+ } else {
+ elapsed += MILLI_SECOND / get_source_fps(encoder);
+ }
+ return elapsed ? encoder->stat_sum * 8 * MILLI_SECOND / elapsed : 0;
+}
+
+static uint64_t get_average_frame_size(GstEncoder *encoder)
+{
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? GSTE_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ return encoder->stat_sum / count;
+}
+
+static uint32_t get_maximum_frame_size(GstEncoder *encoder)
+{
+ if (encoder->stat_maximum == 0) {
+ uint32_t index = encoder->history_last;
+ while (1) {
+ encoder->stat_maximum = MAX(encoder->stat_maximum,
+ encoder->history[index].size);
+ if (index == encoder->stat_first) {
+ break;
+ }
+ index = (index ? index : GSTE_HISTORY_SIZE) - 1;
+ }
+ }
+ return encoder->stat_maximum;
+}
+
+/* Returns the bit rate of the specified period. from and to must be the
+ * mm time of the first and last frame to consider.
+ */
+static uint64_t get_period_bit_rate(GstEncoder *encoder, uint32_t from, uint32_t to)
+{
+ uint32_t sum = 0;
+ uint32_t last_mm_time = 0;
+ uint32_t index = encoder->history_last;
+ while (1) {
+ if (encoder->history[index].mm_time == to) {
+ if (last_mm_time == 0) {
+ /* We don't know how much time elapsed between the period's
+ * last frame and the next so we cannot include it.
+ */
+ sum = 1;
+ last_mm_time = to;
+ } else {
+ sum = encoder->history[index].size + 1;
+ }
+
+ } else if (encoder->history[index].mm_time == from) {
+ sum += encoder->history[index].size;
+ return (sum - 1) * 8 * MILLI_SECOND / (last_mm_time - from);
+
+ } else if (index == encoder->history_first) {
+ /* This period is outside the recorded history */
+ spice_debug("period (%u-%u) outside known history (%u-%u)",
+ from, to,
+ encoder->history[encoder->history_first].mm_time,
+ encoder->history[encoder->history_last].mm_time);
+ return 0;
+
+ } else if (sum > 0) {
+ sum += encoder->history[index].size;
+
+ } else {
+ last_mm_time = encoder->history[index].mm_time;
+ }
+ index = (index ? index : GSTE_HISTORY_SIZE) - 1;
+ }
+
+}
+
+static void add_frame(GstEncoder *encoder, uint32_t frame_mm_time, uint32_t size)
+{
+ /* Update the statistics */
+ uint32_t count = encoder->history_last +
+ (encoder->history_last < encoder->stat_first ? GSTE_HISTORY_SIZE : 0) -
+ encoder->stat_first + 1;
+ if (count == GSTE_FRAME_STATISTICS_COUNT) {
+ encoder->stat_sum -= encoder->history[encoder->stat_first].size;
+ if (encoder->stat_maximum == encoder->history[encoder->stat_first].size) {
+ encoder->stat_maximum = 0;
+ }
+ encoder->stat_first = (encoder->stat_first + 1) % GSTE_HISTORY_SIZE;
+ }
+ encoder->stat_sum += size;
+ if (encoder->stat_maximum > 0 && size > encoder->stat_maximum) {
+ encoder->stat_maximum = size;
+ }
+
+ /* Update the frame history */
+ encoder->history_last = (encoder->history_last + 1) % GSTE_HISTORY_SIZE;
+ if (encoder->history_last == encoder->history_first) {
+ encoder->history_first = (encoder->history_first + 1) % GSTE_HISTORY_SIZE;
+ }
+ encoder->history[encoder->history_last].mm_time = frame_mm_time;
+ encoder->history[encoder->history_last].size = size;
+}
+
+
+/* ---------- Encoder bit rate control ---------- */
+
+static uint32_t get_min_playback_delay(GstEncoder *encoder)
+{
+ /* Make sure the delay is large enough to send a large frame (typically an
+ * I frame) and an average frame. This also takes into account the frames
+ * dropped by the encoder bit rate control.
+ */
+ uint32_t size = get_maximum_frame_size(encoder) + get_average_frame_size(encoder);
+ uint32_t send_time = MILLI_SECOND * size * 8 / encoder->bit_rate;
+
+ /* Also factor in the network latency with a margin for jitter. */
+ uint32_t net_latency = get_network_latency(encoder) * (1.0 + GSTE_LATENCY_MARGIN);
+
+ return send_time + net_latency;
+}
+
+static void update_client_playback_delay(GstEncoder *encoder)
+{
+ if (encoder->cbs.update_client_playback_delay) {
+ uint32_t min_delay = get_min_playback_delay(encoder);
+ encoder->cbs.update_client_playback_delay(encoder->cbs_opaque, min_delay);
+ }
+}
+
+static void update_next_frame(GstEncoder *encoder)
+{
+ if (encoder->vbuffer_free >= 0) {
+ encoder->next_frame = 0;
+ return;
+ }
+
+ /* Figure out how many frames to drop to not exceed the current bit rate.
+ * Use nanoseconds to avoid precision loss.
+ */
+ uint64_t delay_ns = -encoder->vbuffer_free * 8 * NANO_SECOND / encoder->bit_rate;
+ uint64_t period_ns = NANO_SECOND / get_source_fps(encoder);
+ uint32_t drops = (delay_ns + period_ns - 1) / period_ns; /* round up */
+ spice_debug("drops=%u vbuffer %d/%d", drops, encoder->vbuffer_free,
+ encoder->vbuffer_size);
+
+ delay_ns = drops * period_ns + period_ns / 2;
+ if (delay_ns > GSTE_MAX_PERIOD) {
+ delay_ns = GSTE_MAX_PERIOD;
+ }
+ encoder->next_frame = get_last_frame_mm_time(encoder) + delay_ns / NANO_MS;
+
+ /* Drops mean a higher delay between encoded frames so update the playback
+ * delay.
+ */
+ update_client_playback_delay(encoder);
+}
+
+
+/* ---------- Network bit rate control ---------- */
+
/* The maximum bit rate we will use for the current video.
*
* This is based on a 10x compression ratio which should be more than enough
@@ -624,10 +873,21 @@ static int gst_encoder_encode_frame(GstEncoder *encoder,
encoder->spice_format = bitmap->format;
encoder->width = width;
encoder->height = height;
- if (encoder->pipeline) {
+ if (encoder->bit_rate == 0) {
+ encoder->history[0].mm_time = frame_mm_time;
+ encoder->bit_rate = encoder->starting_bit_rate;
+ adjust_bit_rate(encoder);
+ encoder->vbuffer_free = 0; /* Slow start */
+ } else if (encoder->pipeline) {
reconfigure_pipeline(encoder);
}
}
+
+ if (frame_mm_time < encoder->next_frame) {
+ /* Drop the frame to limit the outgoing bit rate. */
+ return VIDEO_ENCODER_FRAME_DROP;
+ }
+
if (!encoder->pipeline && !construct_pipeline(encoder, bitmap)) {
return VIDEO_ENCODER_FRAME_UNSUPPORTED;
}
@@ -640,6 +900,13 @@ static int gst_encoder_encode_frame(GstEncoder *encoder,
spice_assert(!encoder->needs_bitmap);
#endif
}
+ if (rc != VIDEO_ENCODER_FRAME_ENCODE_DONE) {
+ return rc;
+ }
+ add_frame(encoder, frame_mm_time, (*buffer)->base.size);
+
+ update_next_frame(encoder);
+
return rc;
}
@@ -651,10 +918,12 @@ static void gst_encoder_client_stream_report(GstEncoder *encoder,
int32_t end_frame_delay,
uint32_t audio_delay)
{
- spice_debug("client report: #frames %u, #drops %d, duration %u video-delay %d audio-delay %u",
- num_frames, num_drops,
- end_frame_mm_time - start_frame_mm_time,
- end_frame_delay, audio_delay);
+ uint64_t period_bit_rate = get_period_bit_rate(encoder, start_frame_mm_time, end_frame_mm_time);
+ spice_debug("client report: %u/%u drops in %ums margins video %3d audio %3u bw %.3f/%.3fMbps",
+ num_drops, num_frames, end_frame_mm_time - start_frame_mm_time,
+ end_frame_delay, audio_delay,
+ get_mbps(period_bit_rate),
+ get_mbps(get_effective_bit_rate(encoder)));
}
static void gst_encoder_notify_server_frame_drop(GstEncoder *encoder)
@@ -664,7 +933,7 @@ static void gst_encoder_notify_server_frame_drop(GstEncoder *encoder)
static uint64_t gst_encoder_get_bit_rate(GstEncoder *encoder)
{
- return encoder->bit_rate;
+ return get_effective_bit_rate(encoder);
}
static void gst_encoder_get_stats(GstEncoder *encoder, VideoEncoderStats *stats)
@@ -673,7 +942,7 @@ static void gst_encoder_get_stats(GstEncoder *encoder, VideoEncoderStats *stats)
spice_assert(encoder != NULL && stats != NULL);
stats->starting_bit_rate = encoder->starting_bit_rate;
- stats->cur_bit_rate = encoder->bit_rate;
+ stats->cur_bit_rate = get_effective_bit_rate(encoder);
/* Use the compression level as a proxy for the quality */
stats->avg_quality = 100.0 - raw_bit_rate / stats->cur_bit_rate;
@@ -686,6 +955,8 @@ GstEncoder *create_gstreamer_encoder(SpiceVideoCodecType codec_type, uint64_t st
{
GstEncoder *encoder;
+ spice_assert(GSTE_FRAME_STATISTICS_COUNT <= GSTE_HISTORY_SIZE);
+
spice_assert(!cbs || (cbs && cbs->get_roundtrip_ms && cbs->get_source_fps));
if (codec_type != SPICE_VIDEO_CODEC_TYPE_MJPEG &&
codec_type != SPICE_VIDEO_CODEC_TYPE_VP8 &&
@@ -710,7 +981,7 @@ GstEncoder *create_gstreamer_encoder(SpiceVideoCodecType codec_type, uint64_t st
encoder->cbs = *cbs;
}
encoder->cbs_opaque = cbs_opaque;
- encoder->bit_rate = encoder->starting_bit_rate = starting_bit_rate;
+ encoder->starting_bit_rate = starting_bit_rate;
/* All the other fields are initialized to zero by spice_new0(). */
--
2.1.4
_______________________________________________
Spice-devel mailing list
Spice-devel@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/spice-devel