[PATCH spice-gtk 9/11] Add the ability to use gstreamer for vp8 decoding.

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Signed-off-by: Jeremy White <jwhite@xxxxxxxxxxxxxxx>
---
 configure.ac               |  21 ++--
 gtk/Makefile.am            |  12 ++-
 gtk/channel-display-gst.c  | 259 +++++++++++++++++++++++++++++++++++++++++++++
 gtk/channel-display-priv.h |  13 +++
 gtk/channel-display.c      |  11 ++
 5 files changed, 307 insertions(+), 9 deletions(-)
 create mode 100644 gtk/channel-display-gst.c

diff --git a/configure.ac b/configure.ac
index cf5a039..3c89787 100644
--- a/configure.ac
+++ b/configure.ac
@@ -321,18 +321,25 @@ AC_SUBST(PULSE_CFLAGS)
 AC_SUBST(PULSE_LIBS)
 
 AS_IF([test "x$with_audio" = "xgstreamer"],
-      [PKG_CHECK_MODULES(GST, gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-audio-1.0, [have_gst=yes], [have_gst=no])],
-      [have_gst=no])
+      [PKG_CHECK_MODULES(GSTAUDIO, gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-audio-1.0, [have_gst_audio=yes], [have_gst_audio=no])],
+      [have_gst_audio=no])
 
-AS_IF([test "x$have_gst" = "xyes"],
+AS_IF([test "x$have_gst_audio" = "xyes"],
       [AC_DEFINE([WITH_GSTAUDIO], 1, [Have GStreamer 1.0?])],
       [AS_IF([test "x$with_audio" = "xgstreamer"],
              [AC_MSG_ERROR([GStreamer 1.0 requested but not found])
       ])
 ])
-AM_CONDITIONAL([WITH_GSTAUDIO], [test "x$have_gst" = "xyes"])
-AC_SUBST(GST_CFLAGS)
-AC_SUBST(GST_LIBS)
+AM_CONDITIONAL([WITH_GSTAUDIO], [test "x$have_gst_audio" = "xyes"])
+AC_SUBST(GSTAUDIO_CFLAGS)
+AC_SUBST(GSTAUDIO_LIBS)
+
+PKG_CHECK_MODULES(GSTVIDEO, gstreamer-1.0 gstreamer-base-1.0 gstreamer-app-1.0 gstreamer-video-1.0, [have_gst_video=yes], [have_gst_video=no])
+AC_SUBST(GSTVIDEO_CFLAGS)
+AC_SUBST(GSTVIDEO_LIBS)
+AS_IF([test "x$have_gst_video" = "xyes"], [AC_DEFINE([HAVE_GSTVIDEO], 1, [Have Gstreamer Video?])])
+AM_CONDITIONAL([HAVE_GSTVIDEO],[test "$have_gst_video" != "no"])
+
 
 AC_CHECK_LIB(jpeg, jpeg_destroy_decompress,
     AC_MSG_CHECKING([for jpeglib.h])
@@ -724,7 +731,7 @@ SPICE_CFLAGS="$SPICE_CFLAGS $WARN_CFLAGS"
 
 AC_SUBST(SPICE_CFLAGS)
 
-SPICE_GLIB_CFLAGS="$PROTOCOL_CFLAGS $PIXMAN_CFLAGS $PULSE_CFLAGS $GST_CFLAGS $GLIB2_CFLAGS $GIO_CFLAGS $GOBJECT2_CFLAGS $SSL_CFLAGS $SASL_CFLAGS"
+SPICE_GLIB_CFLAGS="$PROTOCOL_CFLAGS $PIXMAN_CFLAGS $PULSE_CFLAGS $GSTVIDEO_CFLAGS $GSTAUDIO_CFLAGS $GLIB2_CFLAGS $GIO_CFLAGS $GOBJECT2_CFLAGS $SSL_CFLAGS $SASL_CFLAGS"
 SPICE_GTK_CFLAGS="$SPICE_GLIB_CFLAGS $GTK_CFLAGS "
 
 AC_SUBST(SPICE_GLIB_CFLAGS)
diff --git a/gtk/Makefile.am b/gtk/Makefile.am
index e1d5d93..aa0519c 100644
--- a/gtk/Makefile.am
+++ b/gtk/Makefile.am
@@ -99,7 +99,8 @@ SPICE_COMMON_CPPFLAGS =						\
 	$(DBUS_GLIB_CFLAGS)					\
 	$(SSL_CFLAGS)						\
 	$(SASL_CFLAGS)						\
-	$(GST_CFLAGS)						\
+	$(GSTAUDIO_CFLAGS)					\
+	$(GSTVIDEO_CFLAGS)					\
 	$(SMARTCARD_CFLAGS)					\
 	$(USBREDIR_CFLAGS)					\
 	$(GUDEV_CFLAGS)						\
@@ -213,7 +214,8 @@ libspice_client_glib_2_0_la_LIBADD =					\
 	$(PIXMAN_LIBS)							\
 	$(SSL_LIBS)							\
 	$(PULSE_LIBS)							\
-	$(GST_LIBS)							\
+	$(GSTAUDIO_LIBS)						\
+	$(GSTVIDEO_LIBS)						\
 	$(SASL_LIBS)							\
 	$(SMARTCARD_LIBS)						\
 	$(USBREDIR_LIBS)						\
@@ -346,6 +348,12 @@ libspice_client_glib_2_0_la_SOURCES +=	\
 	$(NULL)
 endif
 
+if HAVE_GSTVIDEO
+libspice_client_glib_2_0_la_SOURCES +=	\
+	channel-display-gst.c		\
+	$(NULL)
+endif
+
 if WITH_PHODAV
 libspice_client_glib_2_0_la_SOURCES +=	\
 	giopipe.c			\
diff --git a/gtk/channel-display-gst.c b/gtk/channel-display-gst.c
new file mode 100644
index 0000000..f368815
--- /dev/null
+++ b/gtk/channel-display-gst.c
@@ -0,0 +1,259 @@
+/* -*- Mode: C; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+/*
+   Copyright (C) 2015 CodeWeavers, Inc
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "config.h"
+
+#include "spice-client.h"
+#include "spice-common.h"
+#include "spice-channel-priv.h"
+
+#include "channel-display-priv.h"
+
+#include <gst/gst.h>
+#include <gst/app/gstappsrc.h>
+#include <gst/app/gstappsink.h>
+
+struct GstreamerDecoder {
+    GstElement *pipeline;
+    GstElement *appsource;
+    GstElement *appsink;
+    GstElement *codec;
+    GstElement *convert;
+};
+
+static gboolean drain_pipeline(GstreamerDecoder *dec)
+{
+    GstSample *sample = NULL;
+    if (gst_app_sink_is_eos(GST_APP_SINK(dec->appsink)))
+        return TRUE;
+
+    do {
+        if (sample)
+            gst_sample_unref(sample);
+        sample = gst_app_sink_pull_sample(GST_APP_SINK(dec->appsink));
+    } while(sample);
+
+    return gst_app_sink_is_eos(GST_APP_SINK(dec->appsink));
+}
+
+static int construct_pipeline(display_stream *st, GstreamerDecoder *dec)
+{
+    int ret;
+    GstCaps *rgb;
+    char *dec_name;
+
+    if (st->codec == SPICE_VIDEO_CODEC_TYPE_VP8)
+        dec_name = "vp8dec";
+    else if (st->codec == SPICE_VIDEO_CODEC_TYPE_MJPEG)
+        dec_name = "jpegdec";
+    else {
+        SPICE_DEBUG("Uknown codec type %d", st->codec);
+        return -1;
+    }
+
+    if (dec->pipeline) {
+        GstPad *pad = gst_element_get_static_pad(dec->codec, "sink");
+        if (! pad) {
+            SPICE_DEBUG("Unable to get codec sink pad to flush the pipe");
+            return -1;
+        }
+        gst_pad_send_event(pad, gst_event_new_eos());
+
+        gst_object_unref(pad);
+        if (! drain_pipeline(dec))
+            return -1;
+
+        gst_bin_remove_many(GST_BIN(dec->pipeline), dec->appsource, dec->codec, dec->appsink, NULL);
+        gst_object_unref(dec->pipeline);
+    }
+
+    dec->appsource = gst_element_factory_make ("appsrc", NULL);
+    dec->appsink = gst_element_factory_make ("appsink", NULL);
+
+    dec->codec = gst_element_factory_make (dec_name, NULL);
+    dec->convert = gst_element_factory_make ("videoconvert", NULL);
+
+    rgb = gst_caps_from_string("video/x-raw,format=BGRx");
+    if (!rgb) {
+        SPICE_DEBUG("Gstreamer error: could not make BGRx caps.");
+        return -1;
+    }
+
+    if (st->codec == SPICE_VIDEO_CODEC_TYPE_VP8) {
+        GstCaps *vp8;
+        vp8 = gst_caps_from_string("video/x-vp8");
+        if (!vp8) {
+            SPICE_DEBUG("Gstreamer error: could not make vp8 caps.");
+            return -1;
+        }
+
+        gst_app_src_set_caps(GST_APP_SRC(dec->appsource), vp8);
+        gst_app_sink_set_caps(GST_APP_SINK(dec->appsink), rgb);
+    }
+
+    dec->pipeline = gst_pipeline_new ("pipeline");
+
+    if (!dec->pipeline || !dec->appsource || !dec->appsink || !dec->codec || !dec->convert) {
+        SPICE_DEBUG("Gstreamer error: not all elements could be created.");
+        return -1;
+    }
+
+    gst_bin_add_many (GST_BIN(dec->pipeline), dec->appsource, dec->codec, dec->convert, dec->appsink, NULL);
+    if (gst_element_link_many(dec->appsource, dec->codec, dec->convert, dec->appsink, NULL) != TRUE) {
+        SPICE_DEBUG("Gstreamer error: could not link all the pieces.");
+        gst_object_unref (dec->pipeline);
+        return -1;
+    }
+
+    ret = gst_element_set_state (dec->pipeline, GST_STATE_PLAYING);
+    if (ret == GST_STATE_CHANGE_FAILURE) {
+        SPICE_DEBUG("Gstreamer error: Unable to set the pipeline to the playing state.");
+        gst_object_unref (dec->pipeline);
+        return -1;
+    }
+
+    return 0;
+}
+
+static GstreamerDecoder *gst_decoder_new(display_stream *st)
+{
+    GstreamerDecoder *dec;
+
+    gst_init(NULL, NULL);
+
+    dec = g_malloc0(sizeof(*dec));
+    if (construct_pipeline(st, dec)) {
+        free(dec);
+        return NULL;
+    }
+
+    return dec;
+}
+
+
+void gst_decoder_destroy(GstreamerDecoder *dec)
+{
+    gst_object_unref(dec->pipeline);
+    // TODO - contemplate calling gst_deinit();
+    dec->pipeline = NULL;
+    free(dec);
+}
+
+
+gboolean push_frame(display_stream *st)
+{
+    uint8_t *data;
+    uint32_t size;
+    gpointer d;
+    GstBuffer *buffer;
+
+    size = stream_get_current_frame(st, &data);
+    if (size == 0)
+        return false;
+
+    // TODO.  Grr.  Seems like a wasted alloc
+    d = g_malloc(size);
+    memcpy(d, data, size);
+
+    buffer = gst_buffer_new_wrapped(d, size);
+
+    if (gst_app_src_push_buffer(GST_APP_SRC(st->gst_dec->appsource), buffer) != GST_FLOW_OK) {
+        SPICE_DEBUG("Error: unable to push frame of size %d", size);
+        return false;
+    }
+
+    // TODO.  Unref buffer?
+
+    return true;
+}
+
+
+static void pull_frame(display_stream *st)
+{
+    int width;
+    int height;
+
+    GstSample *sample;
+    GstBuffer *buffer = NULL;
+    GstCaps *caps;
+    GstStructure *structure;
+    GstMemory *memory;
+    gint caps_width, caps_height;
+
+    sample = gst_app_sink_pull_sample(GST_APP_SINK(st->gst_dec->appsink));
+    if (! sample) {
+        SPICE_DEBUG("Unable to pull sample");
+        return;
+    }
+
+    buffer = gst_sample_get_buffer(sample);
+    memory = gst_buffer_get_all_memory(buffer);
+    caps = gst_sample_get_caps(sample);
+
+    structure = gst_caps_get_structure (caps, 0);
+    gst_structure_get_int(structure, "width", &caps_width);
+    gst_structure_get_int(structure, "height", &caps_height);
+
+    stream_get_dimensions(st, &width, &height);
+
+    if (width != caps_width || height != caps_height) {
+        SPICE_DEBUG("Stream size %dx%x does not match frame size %dx%d",
+            width, height, caps_width, caps_height);
+    }
+    else {
+        GstMapInfo mem_info;
+
+        // TODO seems like poor memory management
+        if (gst_memory_map(memory, &mem_info, GST_MAP_READ)) {
+            st->out_frame = g_malloc0(mem_info.size);
+            memcpy(st->out_frame, mem_info.data, mem_info.size);
+
+            gst_memory_unmap(memory, &mem_info);
+        }
+    }
+
+    gst_memory_unref(memory);
+    gst_sample_unref(sample);
+}
+
+
+G_GNUC_INTERNAL
+void stream_gst_init(display_stream *st)
+{
+    st->gst_dec = gst_decoder_new(st);
+}
+
+G_GNUC_INTERNAL
+void stream_gst_data(display_stream *st)
+{
+    if (! push_frame(st))
+        return;
+
+    pull_frame(st);
+}
+
+G_GNUC_INTERNAL
+void stream_gst_cleanup(display_stream *st)
+{
+    g_free(st->out_frame);
+    st->out_frame = NULL;
+    if (st->gst_dec) {
+        gst_decoder_destroy(st->gst_dec);
+        st->gst_dec = NULL;
+    }
+}
+
diff --git a/gtk/channel-display-priv.h b/gtk/channel-display-priv.h
index 71f5d17..853dd92 100644
--- a/gtk/channel-display-priv.h
+++ b/gtk/channel-display-priv.h
@@ -34,6 +34,9 @@
 
 G_BEGIN_DECLS
 
+#if defined(HAVE_GSTVIDEO)
+typedef struct GstreamerDecoder GstreamerDecoder;
+#endif
 
 typedef struct display_surface {
     guint32                     surface_id;
@@ -71,6 +74,11 @@ typedef struct display_stream {
     struct jpeg_decompress_struct  mjpeg_cinfo;
     struct jpeg_error_mgr          mjpeg_jerr;
 
+#if defined(HAVE_GSTVIDEO)
+    /* gstreamer decoder */
+    struct GstreamerDecoder     *gst_dec;
+#endif
+
     uint8_t                     *out_frame;
     GQueue                      *msgq;
     guint                       timeout;
@@ -108,6 +116,11 @@ void stream_mjpeg_init(display_stream *st);
 void stream_mjpeg_data(display_stream *st);
 void stream_mjpeg_cleanup(display_stream *st);
 
+/* channel-display-gst.c */
+void stream_gst_init(display_stream *st);
+void stream_gst_data(display_stream *st);
+void stream_gst_cleanup(display_stream *st);
+
 G_END_DECLS
 
 #endif // CHANNEL_DISPLAY_PRIV_H_
diff --git a/gtk/channel-display.c b/gtk/channel-display.c
index efe2259..260dde2 100644
--- a/gtk/channel-display.c
+++ b/gtk/channel-display.c
@@ -596,6 +596,9 @@ static void spice_display_channel_reset_capabilities(SpiceChannel *channel)
     if (SPICE_DISPLAY_CHANNEL(channel)->priv->enable_adaptive_streaming) {
         spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_STREAM_REPORT);
     }
+    spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_MULTI_CODEC);
+    spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_CODEC_MJPEG);
+    spice_channel_set_capability(SPICE_CHANNEL(channel), SPICE_DISPLAY_CAP_CODEC_VP8);
 }
 
 static void destroy_surface(gpointer data)
@@ -1000,6 +1003,8 @@ static void display_handle_stream_create(SpiceChannel *channel, SpiceMsgIn *in)
     case SPICE_VIDEO_CODEC_TYPE_MJPEG:
         stream_mjpeg_init(st);
         break;
+    default:
+        stream_gst_init(st);
     }
 }
 
@@ -1124,6 +1129,9 @@ static gboolean display_stream_render(display_stream *st)
         case SPICE_VIDEO_CODEC_TYPE_MJPEG:
             stream_mjpeg_data(st);
             break;
+        default:
+            stream_gst_data(st);
+            break;
         }
 
         if (st->out_frame) {
@@ -1472,6 +1480,9 @@ static void destroy_stream(SpiceChannel *channel, int id)
     case SPICE_VIDEO_CODEC_TYPE_MJPEG:
         stream_mjpeg_cleanup(st);
         break;
+    default:
+        stream_gst_cleanup(st);
+        break;
     }
 
     if (st->msg_clip)
-- 
2.1.4

_______________________________________________
Spice-devel mailing list
Spice-devel@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/spice-devel





[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Security]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [ECOS]     [Asterisk Internet PBX]     [Linux API]     [Monitors]