[spice-html5 5/6] Add support for the vp8 codec type.

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Signed-off-by: Jeremy White <jwhite@xxxxxxxxxxxxxxx>
---
 display.js   | 230 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 enums.js     |   6 ++
 spiceconn.js |   5 +-
 utils.js     |   1 +
 webm.js      |  98 +++++++++++++++++++++++++
 5 files changed, 337 insertions(+), 3 deletions(-)

diff --git a/display.js b/display.js
index 819f8bc..00b6011 100644
--- a/display.js
+++ b/display.js
@@ -542,8 +542,40 @@ SpiceDisplayConn.prototype.process_channel_message = function(msg)
             console.log("Stream " + m.id + " already exists");
         else
             this.streams[m.id] = m;
-        if (m.codec_type != SPICE_VIDEO_CODEC_TYPE_MJPEG)
-            console.log("Unhandled stream codec: " + m.codec_type);
+
+        if (m.codec_type == SPICE_VIDEO_CODEC_TYPE_VP8)
+        {
+            var media = new MediaSource();
+            var v = document.createElement("video");
+            v.src = window.URL.createObjectURL(media);
+
+            v.setAttribute('autoplay', true);
+            v.setAttribute('width', m.stream_width);
+            v.setAttribute('height', m.stream_height);
+
+            var left = m.dest.left;
+            var top = m.dest.top;
+            if (this.surfaces[m.surface_id] !== undefined)
+            {
+                left += this.surfaces[m.surface_id].canvas.offsetLeft;
+                top += this.surfaces[m.surface_id].canvas.offsetTop;
+            }
+            document.getElementById(this.parent.screen_id).appendChild(v);
+            v.setAttribute('style', "position: absolute; top:" + top + "px; left:" + left + "px;");
+
+            media.addEventListener('sourceopen', handle_video_source_open, false);
+            media.addEventListener('sourceended', handle_video_source_ended, false);
+            media.addEventListener('sourceclosed', handle_video_source_closed, false);
+
+            this.streams[m.id].video = v;
+            this.streams[m.id].media = media;
+
+            media.stream = this.streams[m.id];
+            media.spiceconn = this;
+            v.spice_stream = this.streams[m.id];
+        }
+        else if (m.codec_type != SPICE_VIDEO_CODEC_TYPE_MJPEG)
+            console.log("Unhandled stream codec: "+m.codec_type);
         return true;
     }
 
@@ -568,6 +600,9 @@ SpiceDisplayConn.prototype.process_channel_message = function(msg)
         if (this.streams[m.base.id].codec_type === SPICE_VIDEO_CODEC_TYPE_MJPEG)
             process_mjpeg_stream_data(this, m, latency);
 
+        if (this.streams[m.base.id].codec_type === SPICE_VIDEO_CODEC_TYPE_VP8)
+            process_video_stream_data(this.streams[m.base.id], m);
+
         if ("report" in this.streams[m.base.id])
             process_stream_data_report(this, m, mmtime, latency);
 
@@ -601,6 +636,14 @@ SpiceDisplayConn.prototype.process_channel_message = function(msg)
     {
         var m = new SpiceMsgDisplayStreamDestroy(msg.data);
         DEBUG > 1 && console.log(this.type + ": MsgStreamDestroy id" + m.id);
+
+        if (this.streams[m.id].codec_type == SPICE_VIDEO_CODEC_TYPE_VP8)
+        {
+            document.getElementById(this.parent.screen_id).removeChild(this.streams[m.id].video);
+            this.streams[m.id].source_buffer = null;
+            this.streams[m.id].media = null;
+            this.streams[m.id].video = null;
+        }
         this.streams[m.id] = undefined;
         return true;
     }
@@ -971,3 +1014,186 @@ function process_stream_data_report(sc, m, mmtime, latency)
         sc.streams[m.base.id].report.num_drops = 0;
     }
 }
+
+function handle_video_source_open(e)
+{
+    var stream = this.stream;
+    var p = this.spiceconn;
+
+    if (stream.source_buffer)
+        return;
+
+    var s = this.addSourceBuffer(SPICE_VP8_CODEC);
+    if (! s)
+    {
+        p.log_err('Codec ' + SPICE_VP8_CODEC + ' not available.');
+        return;
+    }
+
+    stream.source_buffer = s;
+    s.spiceconn = p;
+    s.stream = stream;
+
+    stream.queue = new Array();
+    stream.start_time = 0;
+    stream.cluster_time = 0;
+
+    listen_for_video_events(stream);
+
+    var h = new webm_Header();
+    var te = new webm_VideoTrackEntry(this.stream.stream_width, this.stream.stream_height);
+    var t = new webm_Tracks(te);
+
+    var mb = new ArrayBuffer(h.buffer_size() + t.buffer_size())
+
+    var b = h.to_buffer(mb);
+    t.to_buffer(mb, b);
+
+    s.addEventListener('error', handle_video_buffer_error, false);
+    s.addEventListener('updateend', handle_append_video_buffer_done, false);
+
+    append_video_buffer(s, mb);
+}
+
+function handle_video_source_ended(e)
+{
+    var p = this.spiceconn;
+    p.log_err('Video source unexpectedly ended.');
+}
+
+function handle_video_source_closed(e)
+{
+    var p = this.spiceconn;
+    p.log_err('Video source unexpectedly closed.');
+}
+
+function append_video_buffer(sb, mb)
+{
+    try
+    {
+        sb.stream.append_okay = false;
+        sb.appendBuffer(mb);
+    }
+    catch (e)
+    {
+        var p = sb.spiceconn;
+        p.log_err("Error invoking appendBuffer: " + e.message);
+    }
+}
+
+function handle_append_video_buffer_done(e)
+{
+    var stream = this.stream;
+
+    if (stream.queue.length > 0)
+    {
+        var mb = stream.queue.shift();
+        append_video_buffer(stream.source_buffer, mb);
+    }
+    else
+    {
+        stream.append_okay = true;
+    }
+}
+
+function handle_video_buffer_error(e)
+{
+    var p = this.spiceconn;
+    p.log_err('source_buffer error ' + e.message);
+}
+
+function video_simple_block(stream, msg, keyframe)
+{
+    var simple = new webm_SimpleBlock(msg.base.multi_media_time - stream.cluster_time, msg.data, keyframe);
+    var mb = new ArrayBuffer(simple.buffer_size());
+    simple.to_buffer(mb);
+
+    if (stream.append_okay)
+        append_video_buffer(stream.source_buffer, mb);
+    else
+        stream.queue.push(mb);
+}
+
+function new_video_cluster(stream, msg)
+{
+    stream.cluster_time = msg.base.multi_media_time;
+    var c = new webm_Cluster(stream.cluster_time - stream.start_time, msg.data);
+
+    var mb = new ArrayBuffer(c.buffer_size());
+    c.to_buffer(mb);
+
+    if (stream.append_okay)
+        append_video_buffer(stream.source_buffer, mb);
+    else
+        stream.queue.push(mb);
+
+    video_simple_block(stream, msg, true);
+}
+
+function process_video_stream_data(stream, msg)
+{
+    if (! stream.source_buffer)
+        return true;
+
+    if (stream.start_time == 0)
+    {
+        stream.start_time = msg.base.multi_media_time;
+        stream.video.play();
+        new_video_cluster(stream, msg);
+    }
+
+    else if (msg.base.multi_media_time - stream.cluster_time >= MAX_CLUSTER_TIME)
+        new_video_cluster(stream, msg);
+    else
+        video_simple_block(stream, msg, false);
+}
+
+function video_handle_event_debug(e)
+{
+    var s = this.spice_stream;
+    if (s.video)
+    {
+        if (STREAM_DEBUG > 0 || s.video.buffered.len > 1)
+            console.log(s.video.currentTime + ":id " +  s.id + " event " + e.type +
+                dump_media_element(s.video));
+    }
+
+    if (STREAM_DEBUG > 1 && s.media)
+        console.log("  media_source " + dump_media_source(s.media));
+
+    if (STREAM_DEBUG > 1 && s.source_buffer)
+        console.log("  source_buffer " + dump_source_buffer(s.source_buffer));
+
+    if (STREAM_DEBUG > 0 || s.queue.length > 1)
+        console.log('  queue len ' + s.queue.length + '; append_okay: ' + s.append_okay);
+}
+
+function video_debug_listen_for_one_event(name)
+{
+    this.addEventListener(name, video_handle_event_debug);
+}
+
+function listen_for_video_events(stream)
+{
+    var video_0_events = [
+        "abort", "error"
+    ];
+
+    var video_1_events = [
+        "loadstart", "suspend", "emptied", "stalled", "loadedmetadata", "loadeddata", "canplay",
+        "canplaythrough", "playing", "waiting", "seeking", "seeked", "ended", "durationchange",
+        "timeupdate", "play", "pause", "ratechange"
+    ];
+
+    var video_2_events = [
+        "progress",
+        "resize",
+        "volumechange"
+    ];
+
+    video_0_events.forEach(video_debug_listen_for_one_event, stream.video);
+    if (STREAM_DEBUG > 0)
+        video_1_events.forEach(video_debug_listen_for_one_event, stream.video);
+    if (STREAM_DEBUG > 1)
+        video_2_events.forEach(video_debug_listen_for_one_event, stream.video);
+}
diff --git a/enums.js b/enums.js
index bca463a..3ef36dc 100644
--- a/enums.js
+++ b/enums.js
@@ -182,6 +182,11 @@ var SPICE_DISPLAY_CAP_COMPOSITE           = 2;
 var SPICE_DISPLAY_CAP_A8_SURFACE          = 3;
 var SPICE_DISPLAY_CAP_STREAM_REPORT       = 4;
 var SPICE_DISPLAY_CAP_LZ4_COMPRESSION     = 5;
+var SPICE_DISPLAY_CAP_PREF_COMPRESSION    = 6;
+var SPICE_DISPLAY_CAP_GL_SCANOUT          = 7;
+var SPICE_DISPLAY_CAP_MULTI_CODEC         = 8;
+var SPICE_DISPLAY_CAP_CODEC_MJPEG         = 9;
+var SPICE_DISPLAY_CAP_CODEC_VP8           = 10;
 
 var SPICE_AUDIO_DATA_MODE_INVALID       = 0;
 var SPICE_AUDIO_DATA_MODE_RAW           = 1;
@@ -324,6 +329,7 @@ var SPICE_CURSOR_TYPE_ALPHA     = 0,
     SPICE_CURSOR_TYPE_COLOR32   = 6;
 
 var SPICE_VIDEO_CODEC_TYPE_MJPEG = 1;
+var SPICE_VIDEO_CODEC_TYPE_VP8   = 2;
 
 var VD_AGENT_PROTOCOL = 1;
 var VD_AGENT_MAX_DATA_SIZE = 2048;
diff --git a/spiceconn.js b/spiceconn.js
index f20424f..41d2fa3 100644
--- a/spiceconn.js
+++ b/spiceconn.js
@@ -136,7 +136,10 @@ SpiceConn.prototype =
         else if (msg.channel_type == SPICE_CHANNEL_DISPLAY)
             msg.channel_caps.push(
                 (1 << SPICE_DISPLAY_CAP_SIZED_STREAM) |
-                (1 << SPICE_DISPLAY_CAP_STREAM_REPORT)
+                (1 << SPICE_DISPLAY_CAP_STREAM_REPORT) |
+                (1 << SPICE_DISPLAY_CAP_MULTI_CODEC) |
+                (1 << SPICE_DISPLAY_CAP_CODEC_MJPEG) |
+                (1 << SPICE_DISPLAY_CAP_CODEC_VP8)
             );
 
         hdr.size = msg.buffer_size();
diff --git a/utils.js b/utils.js
index 2583666..9093a24 100644
--- a/utils.js
+++ b/utils.js
@@ -23,6 +23,7 @@
 **--------------------------------------------------------------------------*/
 var DEBUG = 0;
 var PLAYBACK_DEBUG = 0;
+var STREAM_DEBUG = 0;
 var DUMP_DRAWS = false;
 var DUMP_CANVASES = false;
 
diff --git a/webm.js b/webm.js
index 7d27b86..8faa8e7 100644
--- a/webm.js
+++ b/webm.js
@@ -61,6 +61,10 @@ var WEBM_CODEC_DELAY =                      [ 0x56, 0xAA ];
 var WEBM_CODEC_PRIVATE =                    [ 0x63, 0xA2 ];
 var WEBM_CODEC_ID =                         [ 0x86 ];
 
+var WEBM_VIDEO =                            [ 0xE0 ] ;
+var WEBM_PIXEL_WIDTH =                      [ 0xB0 ] ;
+var WEBM_PIXEL_HEIGHT =                     [ 0xBA ] ;
+
 var WEBM_AUDIO =                            [ 0xE1 ] ;
 var WEBM_SAMPLING_FREQUENCY =               [ 0xB5 ] ;
 var WEBM_CHANNELS =                         [ 0x9F ] ;
@@ -82,6 +86,8 @@ var MAX_CLUSTER_TIME                        = 1000;
 
 var GAP_DETECTION_THRESHOLD                 = 50;
 
+var SPICE_VP8_CODEC                         = 'video/webm; codecs="vp8"';
+
 /*----------------------------------------------------------------------------
 **  EBML utility functions
 **      These classes can create the binary representation of a webm file
@@ -291,6 +297,34 @@ webm_Audio.prototype =
     },
 }
 
+function webm_Video(width, height)
+{
+    this.id = WEBM_VIDEO;
+    this.width = width;
+    this.height = height;
+}
+
+webm_Video.prototype =
+{
+    to_buffer: function(a, at)
+    {
+        at = at || 0;
+        var dv = new DataView(a);
+        at = EBML_write_array(this.id, dv, at);
+        at = EBML_write_u64_data_len(this.buffer_size() - 8 - this.id.length, dv, at);
+        at = EBML_write_u16_value(WEBM_PIXEL_WIDTH, this.width, dv, at)
+        at = EBML_write_u16_value(WEBM_PIXEL_HEIGHT, this.height, dv, at)
+        return at;
+    },
+    buffer_size: function()
+    {
+        return this.id.length + 8 +
+            WEBM_PIXEL_WIDTH.length + 1 + 2 +
+            WEBM_PIXEL_HEIGHT.length + 1 + 2;
+    },
+}
+
+
 
 /* ---------------------------
    SeekHead not currently used.  Hopefully not needed.
@@ -431,6 +465,70 @@ webm_AudioTrackEntry.prototype =
             this.audio.buffer_size();
     },
 }
+
+function webm_VideoTrackEntry(width, height)
+{
+    this.id = WEBM_TRACK_ENTRY;
+    this.number = 1;
+    this.uid = 1;
+    this.type = 1; // Video
+    this.flag_enabled = 1;
+    this.flag_default = 1;
+    this.flag_forced = 1;
+    this.flag_lacing = 0;
+    this.min_cache = 0; // fixme - check
+    this.max_block_addition_id = 0;
+    this.codec_decode_all = 0; // fixme - check
+    this.seek_pre_roll = 0; // 80000000; // fixme - check
+    this.codec_delay =   80000000; // Must match codec_private.preskip
+    this.codec_id = "V_VP8";
+    this.video = new webm_Video(width, height);
+}
+
+webm_VideoTrackEntry.prototype =
+{
+    to_buffer: function(a, at)
+    {
+        at = at || 0;
+        var dv = new DataView(a);
+        at = EBML_write_array(this.id, dv, at);
+        at = EBML_write_u64_data_len(this.buffer_size() - 8 - this.id.length, dv, at);
+        at = EBML_write_u8_value(WEBM_TRACK_NUMBER, this.number, dv, at);
+        at = EBML_write_u8_value(WEBM_TRACK_UID, this.uid, dv, at);
+        at = EBML_write_u8_value(WEBM_FLAG_ENABLED, this.flag_enabled, dv, at);
+        at = EBML_write_u8_value(WEBM_FLAG_DEFAULT, this.flag_default, dv, at);
+        at = EBML_write_u8_value(WEBM_FLAG_FORCED, this.flag_forced, dv, at);
+        at = EBML_write_u8_value(WEBM_FLAG_LACING, this.flag_lacing, dv, at);
+        at = EBML_write_data(WEBM_CODEC_ID, this.codec_id, dv, at);
+        at = EBML_write_u8_value(WEBM_MIN_CACHE, this.min_cache, dv, at);
+        at = EBML_write_u8_value(WEBM_MAX_BLOCK_ADDITION_ID, this.max_block_addition_id, dv, at);
+        at = EBML_write_u8_value(WEBM_CODEC_DECODE_ALL, this.codec_decode_all, dv, at);
+        at = EBML_write_u32_value(WEBM_CODEC_DELAY, this.codec_delay, dv, at);
+        at = EBML_write_u32_value(WEBM_SEEK_PRE_ROLL, this.seek_pre_roll, dv, at);
+        at = EBML_write_u8_value(WEBM_TRACK_TYPE, this.type, dv, at);
+        at = this.video.to_buffer(a, at);
+        return at;
+    },
+    buffer_size: function()
+    {
+        return this.id.length + 8 +
+            WEBM_TRACK_NUMBER.length + 1 + 1 +
+            WEBM_TRACK_UID.length + 1 + 1 +
+            WEBM_FLAG_ENABLED.length + 1 + 1 +
+            WEBM_FLAG_DEFAULT.length + 1 + 1 +
+            WEBM_FLAG_FORCED.length + 1 + 1 +
+            WEBM_FLAG_LACING.length + 1 + 1 +
+            WEBM_CODEC_ID.length + this.codec_id.length + 1 +
+            WEBM_MIN_CACHE.length + 1 + 1 +
+            WEBM_MAX_BLOCK_ADDITION_ID.length + 1 + 1 +
+            WEBM_CODEC_DECODE_ALL.length + 1 + 1 +
+            WEBM_CODEC_DELAY.length + 1 + 4 +
+            WEBM_SEEK_PRE_ROLL.length + 1 + 4 +
+            WEBM_TRACK_TYPE.length + 1 + 1 +
+            this.video.buffer_size();
+    },
+}
+
 function webm_Tracks(entry)
 {
     this.id = WEBM_TRACKS;
-- 
2.1.4

_______________________________________________
Spice-devel mailing list
Spice-devel@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/spice-devel




[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Security]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [ECOS]     [Asterisk Internet PBX]     [Linux API]     [Monitors]