Gstreamer based plugin utilizing gstreamer elements to capture
screen from X, convert and encode into h264/h265/vp8/vp9/mjpeg stream
Configure with --enable-gstreamer, will be built as a separate plugin.
The plugin was made for testing purposes, it was mainly tested with
the x264enc (h264 is the defualt codec) encoder.
To choose codec type use: '-c gst.codec=<h264/h265/vp8/vp9/mjpeg>'
To specify a certain plugin use: '-c gst.encoder=<plugin name>' in
addition to its matching codec type (gst.codec).
---
configure.ac | 15 ++
src/Makefile.am | 27 +++
src/gst-plugin.cpp | 435 +++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 477 insertions(+)
create mode 100644 src/gst-plugin.cpp
diff --git a/configure.ac b/configure.ac
index b59c447..b730bb2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -60,6 +60,20 @@ AC_ARG_WITH(udevrulesdir,
)
AC_SUBST(UDEVRULESDIR)
+AC_ARG_ENABLE(gstreamer,
+ AS_HELP_STRING([--enable-gstreamer=@<:@auto/yes/no@:>@],
+ [Enable GStreamer support]),,
+ [enable_gstreamer="no"])
+if test "$enable_gstreamer" != "no"; then
+ PKG_CHECK_MODULES(GST, [gstreamer-1.0 gstreamer-app-1.0], [enable_gstreamer=yes],
+ [if test "$enable_gstreamer" = "yes"; then
+ AC_MSG_ERROR([Gstreamer libs are missing])
+ fi
+ enable_gstreamer=no
+ ])
+fi
+AM_CONDITIONAL([HAVE_GST],[test "$enable_gstreamer" = "yes"])
+
dnl ===========================================================================
dnl check compiler flags
@@ -129,6 +143,7 @@ AC_MSG_NOTICE([
prefix: ${prefix}
C compiler: ${CC}
C++ compiler: ${CXX}
+ Gstreamer plugin: ${enable_gstreamer}
Now type 'make' to build $PACKAGE
])
diff --git a/src/Makefile.am b/src/Makefile.am
index fead680..febb688 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -9,6 +9,9 @@ if ENABLE_TESTS
SUBDIRS = . unittests
endif
+plugin_LTLIBRARIES =
+plugindir = $(pkglibdir)/plugins
+
AM_CPPFLAGS = \
-DSPICE_STREAMING_AGENT_PROGRAM \
-I$(top_srcdir)/include \
@@ -63,3 +66,27 @@ spice_streaming_agent_SOURCES = \
stream-port.cpp \
stream-port.hpp \
$(NULL)
+
+if HAVE_GST
+plugin_LTLIBRARIES += gst-plugin.la
+
+gst_plugin_la_LDFLAGS = \
+ -module -avoid-version \
+ $(RELRO_LDFLAGS) \
+ $(NO_INDIRECT_LDFLAGS) \
+ $(NULL)
+
+gst_plugin_la_LIBADD = \
+ $(GST_LIBS) \
+ $(NULL)
+
+gst_plugin_la_SOURCES = \
+ gst-plugin.cpp \
+ $(NULL)
+
+gst_plugin_la_CPPFLAGS = \
+ -I$(top_srcdir)/include \
+ $(SPICE_PROTOCOL_CFLAGS) \
+ $(GST_CFLAGS) \
+ $(NULL)
+endif
diff --git a/src/gst-plugin.cpp b/src/gst-plugin.cpp
new file mode 100644
index 0000000..afab0e7
--- /dev/null
+++ b/src/gst-plugin.cpp
@@ -0,0 +1,435 @@
+/* Plugin implementation for gstreamer encoder
+ *
+ * \copyright
+ * Copyright 2018 Red Hat Inc. All rights reserved.
+ */
+
+#include <config.h>
+#include <cstring>
+#include <exception>
+#include <stdexcept>
+#include <sstream>
+#include <memory>
+#include <syslog.h>
+#include <unistd.h>
+#include <gst/gst.h>
+#include <gst/app/gstappsink.h>
+
+#define XLIB_CAPTURE 1
+#if XLIB_CAPTURE
+#include <X11/Xlib.h>
+#include <gst/app/gstappsrc.h>
+#endif
+
+#include <spice-streaming-agent/plugin.hpp>
+#include <spice-streaming-agent/frame-capture.hpp>
+
+#define gst_syslog(priority, str, ...) syslog(priority, "Gstreamer plugin: " str, ## __VA_ARGS__);
+
+namespace spice {
+namespace streaming_agent {
+namespace gstreamer_plugin {
+
+struct GstreamerEncoderSettings
+{
+ int fps;
+ SpiceVideoCodecType codec;
+ std::string encoder;
+};
+
+class GstreamerFrameCapture final: public FrameCapture
+{
+public:
+ GstreamerFrameCapture(const GstreamerEncoderSettings &settings);
+ ~GstreamerFrameCapture();
+ FrameInfo CaptureFrame() override;
+ void Reset() override;
+ SpiceVideoCodecType VideoCodecType() const override {
+ return settings.codec;
+ }
+private:
+ void free_sample();
+ GstElement *get_encoder_plugin(const GstreamerEncoderSettings& settings);
+ GstElement *get_capture_plugin(const GstreamerEncoderSettings& settings);
+ void pipeline_init(const GstreamerEncoderSettings& settings);
+#if XLIB_CAPTURE
+ void xlib_capture();
+ Display *dpy;
+ XImage *image = nullptr;
+#endif
+ GstElement *pipeline = nullptr;
+ GstElement *capture, *sink, *encoder, *convert;
+ GstSample *sample = nullptr;
+ GstCaps *sink_caps = nullptr;
+ GstMapInfo map = {};
+ uint32_t last_width = ~0u, last_height = ~0u;
+ uint32_t cur_width = 0, cur_height = 0;
+ bool is_first = true;
+ GstreamerEncoderSettings settings; // will be set by plugin settings
+};
+
+class GstreamerPlugin final: public Plugin
+{
+public:
+ FrameCapture *CreateCapture() override;
+ unsigned Rank() override;
+ void ParseOptions(const ConfigureOption *options);
+ SpiceVideoCodecType VideoCodecType() const override {
+ return settings.codec;
+ }
+private:
+ GstreamerEncoderSettings settings = { .fps = 25, .codec = SPICE_VIDEO_CODEC_TYPE_H264 };
+};
+
+GstElement *GstreamerFrameCapture::get_capture_plugin(const GstreamerEncoderSettings& settings)
+{
+ GstElement *capture = nullptr;
+
+#if XLIB_CAPTURE
+ capture = gst_element_factory_make("appsrc", "capture");
+#else
+ capture = gst_element_factory_make("ximagesrc", "capture");
+ g_object_set(capture,
+ "use-damage", 0,
+ NULL);
+
+#endif
+ return capture;
+}
+
+GstElement *GstreamerFrameCapture::get_encoder_plugin(const GstreamerEncoderSettings& settings)
+{
+ GList *encoders;
+ GList *filtered;
+ GstElement *encoder;
+ GstElementFactory *factory = nullptr;
+ std::stringstream caps_ss;
+
+ switch (settings.codec) {
+ case SPICE_VIDEO_CODEC_TYPE_H264:
+ caps_ss << "video/x-h264" << ",stream-format=(string)byte-stream";
+ break;
+ case SPICE_VIDEO_CODEC_TYPE_MJPEG:
+ caps_ss << "image/jpeg";
+ break;
+ case SPICE_VIDEO_CODEC_TYPE_VP8:
+ caps_ss << "video/x-vp8";
+ break;
+ case SPICE_VIDEO_CODEC_TYPE_VP9:
+ caps_ss << "video/x-vp9";
+ break;
+ case SPICE_VIDEO_CODEC_TYPE_H265:
+ caps_ss << "video/x-h265";
+ break;
+ default : /* Should not happen - just to avoid compiler's complaint */
+ return nullptr;
+ }
+ caps_ss << ",framerate=" << settings.fps << "/1";
+
+ encoders = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_VIDEO_ENCODER, GST_RANK_NONE);
+ sink_caps = gst_caps_from_string(caps_ss.str().c_str());
+ filtered = gst_element_factory_list_filter(encoders, sink_caps, GST_PAD_SRC, false);
+ if (filtered) {
+ gst_syslog(LOG_NOTICE, "Looking for plugins which can produce a '%s' stream", caps_ss.str().c_str());
+ for (GList *l = filtered; l != NULL; l = l->next) {
+ if (!factory && !settings.encoder.compare(GST_ELEMENT_NAME(l->data))) {
+ factory = (GstElementFactory*)l->data;
+ }
+ gst_syslog(LOG_NOTICE, "'%s' plugin is available", GST_ELEMENT_NAME(l->data));
+ }
+ if (!factory && !settings.encoder.empty()) {
+ gst_syslog(LOG_WARNING, "Specified encoder '%s' cannot produce '%s' stream, make sure matching gst.codec is specified", settings.encoder.c_str(), caps_ss.str().c_str());
+ }
+ factory = factory ? factory : (GstElementFactory*)filtered->data;
+ gst_syslog(LOG_NOTICE, "'%s' encoder plugin is used", GST_ELEMENT_NAME(factory));
+
+ } else {
+ gst_syslog(LOG_ERR, "No suitable encoder was found for '%s'", caps_ss.str().c_str());
+ }
+
+ encoder = factory ? gst_element_factory_create(factory, "encoder") : nullptr;
+ if (encoder) { // Invalid properties will be ignored silently
+ /* x264enc properties */
+ gst_util_set_object_arg(G_OBJECT(encoder), "tune", "zerolatency");// stillimage, fastdecode, zerolatency
+ gst_util_set_object_arg(G_OBJECT(encoder), "bframes", "0");
+ gst_util_set_object_arg(G_OBJECT(encoder), "speed-preset", "1");// 1-ultrafast, 6-med, 9-veryslow
+ }
+ gst_plugin_feature_list_free(filtered);
+ gst_plugin_feature_list_free(encoders);
+ return encoder;
+}
+
+void GstreamerFrameCapture::pipeline_init(const GstreamerEncoderSettings& settings)
+{
+ GstElement *encoder, *convert;
+ GstCaps *caps;
+ gboolean link;
+
+ if (!(pipeline = gst_pipeline_new("pipeline"))) {
+ throw std::runtime_error("Gstreamer's pipeline element cannot be created");
+ }
+ if (!(capture = get_capture_plugin(settings))) {
+ throw std::runtime_error("Gstreamer's capture element cannot be created");
+ }
+ if (!(convert = gst_element_factory_make("videoconvert", "convert"))) {
+ gst_object_unref(capture);
+ throw std::runtime_error("Gstreamer's 'videoconvert' element cannot be created");
+ }
+ if (!(encoder = get_encoder_plugin(settings))) {
+ gst_object_unref(capture);
+ gst_object_unref(convert);
+ throw std::runtime_error("Gstreamer's encoder element cannot be created");
+ }
+ if (!(sink = gst_element_factory_make("appsink", "sink"))) {
+ gst_object_unref(capture);
+ gst_object_unref(convert);
+ gst_object_unref(encoder);
+ throw std::runtime_error("Gstreamer's appsink element cannot be created");
+ }
+
+ g_object_set(sink,
+ "sync", FALSE,
+ "drop", TRUE,
+ "max-buffers", 1,
+ NULL);
+
+ gst_bin_add_many(GST_BIN(pipeline), capture, convert, encoder, sink, NULL);
+
+ //caps = gst_caps_from_string("video/x-raw,format=(string)I420");
+ caps = gst_caps_from_string("video/x-raw");
+ link = gst_element_link(capture, convert) &&
+ gst_element_link_filtered(convert, encoder, caps) &&
+ gst_element_link_filtered(encoder, sink, sink_caps);
+ gst_caps_unref(caps);
+ if (!link) {
+ throw std::runtime_error("Linking gstreamer's elements failed");
+ }
+
+#if XLIB_CAPTURE
+ dpy = XOpenDisplay(NULL);
+ if (!dpy) {
+ throw std::runtime_error("Unable to initialize X11");
+ }
+#endif
+
+ gst_element_set_state(pipeline, GST_STATE_PLAYING);
+
+#if !XLIB_CAPTURE
+ int sx, sy, ex, ey;
+ g_object_get(capture,
+ "endx", &ex,
+ "endy", &ey,
+ "startx", &sx,
+ "starty", &sy,
+ NULL);
+ cur_width = ex - sx;
+ cur_height = ey - sy;
+ if (cur_width < 16 || cur_height < 16) {
+ throw std::runtime_error("Invalid screen size");
+ }
+ g_object_set(capture,
+ /* Some encoders cannot handle odd resolution make sure it's even number of pixels
+ * pixel counting starts from zero here!
+ */
+ "endx", cur_width - !(cur_width%2),
+ "endy", cur_height - !(cur_height%2),
+ "startx", 0,
+ "starty", 0,
+ NULL);
+#endif
+}
+
+GstreamerFrameCapture::GstreamerFrameCapture(const GstreamerEncoderSettings& settings):
+ settings(settings)
+{
+ try {
+ pipeline_init(settings);
+ } catch (const std::exception &e) {
+ if (pipeline) {
+ gst_object_unref(pipeline);
+ }
+ if (sink_caps) {
+ gst_caps_unref(sink_caps);
+ }
+ throw;
+ }
+}
+
+void GstreamerFrameCapture::free_sample()
+{
+ if (sample) {
+ gst_buffer_unmap(gst_sample_get_buffer(sample), &map);
+ gst_sample_unref(sample);
+ sample = nullptr;
+ }
+#if XLIB_CAPTURE
+ if(image) {
+ image->f.destroy_image(image);
+ image = nullptr;
+ }
+#endif
+}
+
+GstreamerFrameCapture::~GstreamerFrameCapture()
+{
+ free_sample();
+ gst_element_set_state(pipeline, GST_STATE_NULL);
+ gst_object_unref(pipeline);
+ gst_caps_unref(sink_caps);
+#if XLIB_CAPTURE
+ XCloseDisplay(dpy);
+#endif
+}
+
+void GstreamerFrameCapture::Reset()
+{
+ //TODO
+}
+
+#if XLIB_CAPTURE
+void GstreamerFrameCapture::xlib_capture()
+{
+ int screen = XDefaultScreen(dpy);
+
+ Window win = RootWindow(dpy, screen);
+ XWindowAttributes win_info;
+ XGetWindowAttributes(dpy, win, &win_info);
+
+ /* Some encoders cannot handle odd resolution make sure it's even number of pixels */
+ cur_width = win_info.width - win_info.width%2;
+ cur_height = win_info.height - win_info.height%2;
+
+ if (cur_width != last_width || cur_height != last_height) {
+ last_width = cur_width;
+ last_height = cur_height;
+ is_first = true;
+
+ gst_app_src_end_of_stream (GST_APP_SRC (capture));
+ gst_element_set_state(pipeline, GST_STATE_NULL);//maybe ximagesrc needs eos as well
+ gst_element_set_state(pipeline, GST_STATE_PLAYING);
+ }
+
+ image = XGetImage(dpy, win, 0, 0,
+ cur_width, cur_height, AllPlanes, ZPixmap);
+ if (!image) {
+ throw std::runtime_error("Cannot capture from X");
+ }
+
+ GstBuffer *buf;
+ buf = gst_buffer_new_wrapped(image->data, image->height * image->bytes_per_line);
+ if (!buf) {
+ throw std::runtime_error("Failed to wrap image in gstreamer buffer");
+ }
+
+ GstSample *appsrc_sample;
+ GstCaps *caps;
+ std::stringstream ss;
+
+ ss << "video/x-raw,format=BGRx,width=" << image->width << ",height=" << image->height << ",framerate=" << settings.fps << "/1";
+ caps = gst_caps_from_string(ss.str().c_str());
+
+ // Push sample
+ appsrc_sample = gst_sample_new(buf, caps, NULL, NULL);
+ if (gst_app_src_push_sample (GST_APP_SRC (capture), appsrc_sample) != GST_FLOW_OK) {
+ throw std::runtime_error("gstramer appsrc element cannot push sample");
+ }
+ gst_sample_unref(appsrc_sample);
+}
+#endif
+
+FrameInfo GstreamerFrameCapture::CaptureFrame()
+{
+ FrameInfo info;
+
+ free_sample(); // free prev if exist
+
+#if XLIB_CAPTURE
+ xlib_capture();
+#endif
+ info.size.width = cur_width;
+ info.size.height = cur_height;
+ info.stream_start = is_first;
+ if (is_first) {
+ is_first = false;
+ }
+
+ // Pull sample
+ sample = gst_app_sink_pull_sample(GST_APP_SINK(sink)); // blocking
+
+ if (sample) { // map after pipeline
+ if (!gst_buffer_map(gst_sample_get_buffer(sample), &map, GST_MAP_READ)) {
+ free_sample();
+ throw std::runtime_error("Buffer mapping failed");
+ }
+
+ info.buffer = map.data;
+ info.buffer_size = map.size;
+ } else {
+ throw std::runtime_error("No sample- EOS or state change");
+ }
+
+ return info;
+}
+
+FrameCapture *GstreamerPlugin::CreateCapture()
+{
+ return new GstreamerFrameCapture(settings);
+}
+
+unsigned GstreamerPlugin::Rank()
+{
+ return SoftwareMin;
+}
+
+void GstreamerPlugin::ParseOptions(const ConfigureOption *options)
+{
+ for (; options->name; ++options) {
+ const std::string name = options->name;
+ const std::string value = options->value;
+
+ if (name == "framerate") {
+ try {
+ settings.fps = std::stoi(value);
+ } catch (const std::exception &e) {
+ throw std::runtime_error("Invalid value '" + value + "' for option 'framerate'.");
+ }
+ } else if (name == "gst.codec") {
+ if (value == "h264") {
+ settings.codec = SPICE_VIDEO_CODEC_TYPE_H264;
+ } else if (value == "vp9") {
+ settings.codec = SPICE_VIDEO_CODEC_TYPE_VP9;
+ } else if (value == "vp8") {
+ settings.codec = SPICE_VIDEO_CODEC_TYPE_VP8;
+ } else if (value == "mjpeg") {
+ settings.codec = SPICE_VIDEO_CODEC_TYPE_MJPEG;
+ } else if (value == "h265") {
+ settings.codec = SPICE_VIDEO_CODEC_TYPE_H265;
+ } else {
+ throw std::runtime_error("Invalid value '" + value + "' for option 'gst.codec'.");
+ }
+ } else if (name == "gst.encoder") {
+ if (value.length() < 3) {
+ gst_syslog(LOG_WARNING, "Encoder name length is invalid, will be ignored");
+ } else {
+ settings.encoder = value;
+ }
+ }
+ }
+}
+
+}}} //namespace spice::streaming_agent::gstreamer_plugin
+
+using namespace spice::streaming_agent::gstreamer_plugin;
+
+SPICE_STREAMING_AGENT_PLUGIN(agent)
+{
+ gst_init(NULL, NULL);
+
+ std::unique_ptr<GstreamerPlugin> plugin(new GstreamerPlugin());
+
+ plugin->ParseOptions(agent->Options());
+
+ agent->Register(*plugin.release());
+
+ return true;
+}
--
2.17.0
_______________________________________________
Spice-devel mailing list
Spice-devel@xxxxxxxxxxxxxxxxxxxxx
https://lists.freedesktop.org/mailman/listinfo/spice-devel