On 16/08/18 11:59, Snir Sheriber wrote: > Gstreamer based plugin utilizing gstreamer elements to capture > screen from X, convert and encode into h264/h265/vp8/vp9/mjpeg stream > Configure with --enable-gstreamer, will be built as a separate plugin. > Did not look at the code, but I would rather go with something like --enable-gst-plugin instead. --enable-gstreamer may cause confusion, as this terminology is widely adopted with the meaning of building with/without gstreamer. > The plugin was made for testing purposes, it was mainly tested with > the x264enc (h264 is the defualt codec) encoder. > To choose codec type use: '-c gst.codec=<h264/h265/vp8/vp9/mjpeg>' > To specify a certain plugin use: '-c gst.encoder=<plugin name>' in > addition to its matching codec type (gst.codec). > > Signed-off-by: Snir Sheriber <ssheribe@xxxxxxxxxx> > Signed-off-by: Frediano Ziglio <fziglio@xxxxxxxxxx> > --- > configure.ac | 15 ++ > src/Makefile.am | 27 +++ > src/gst-plugin.cpp | 466 +++++++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 508 insertions(+) > create mode 100644 src/gst-plugin.cpp > > diff --git a/configure.ac b/configure.ac > index b59c447..b730bb2 100644 > --- a/configure.ac > +++ b/configure.ac > @@ -60,6 +60,20 @@ AC_ARG_WITH(udevrulesdir, > ) > AC_SUBST(UDEVRULESDIR) > > +AC_ARG_ENABLE(gstreamer, > + AS_HELP_STRING([--enable-gstreamer=@<:@auto/yes/no@:>@], > + [Enable GStreamer support]),, > + [enable_gstreamer="no"]) > +if test "$enable_gstreamer" != "no"; then > + PKG_CHECK_MODULES(GST, [gstreamer-1.0 gstreamer-app-1.0], [enable_gstreamer=yes], > + [if test "$enable_gstreamer" = "yes"; then > + AC_MSG_ERROR([Gstreamer libs are missing]) > + fi > + enable_gstreamer=no > + ]) > +fi > +AM_CONDITIONAL([HAVE_GST],[test "$enable_gstreamer" = "yes"]) > + > dnl =========================================================================== > dnl check compiler flags > > @@ -129,6 +143,7 @@ AC_MSG_NOTICE([ > prefix: ${prefix} > C compiler: ${CC} > C++ compiler: ${CXX} > + Gstreamer plugin: ${enable_gstreamer} > > Now type 'make' to build $PACKAGE > ]) > diff --git a/src/Makefile.am b/src/Makefile.am > index 40544ba..104da47 100644 > --- a/src/Makefile.am > +++ b/src/Makefile.am > @@ -9,6 +9,9 @@ if ENABLE_TESTS > SUBDIRS = . unittests > endif > > +plugin_LTLIBRARIES = > +plugindir = $(pkglibdir)/plugins > + > AM_CPPFLAGS = \ > -DSPICE_STREAMING_AGENT_PROGRAM \ > -I$(top_srcdir)/include \ > @@ -65,3 +68,27 @@ spice_streaming_agent_SOURCES = \ > stream-port.cpp \ > stream-port.hpp \ > $(NULL) > + > +if HAVE_GST > +plugin_LTLIBRARIES += gst-plugin.la > + > +gst_plugin_la_LDFLAGS = \ > + -module -avoid-version \ > + $(RELRO_LDFLAGS) \ > + $(NO_INDIRECT_LDFLAGS) \ > + $(NULL) > + > +gst_plugin_la_LIBADD = \ > + $(GST_LIBS) \ > + $(NULL) > + > +gst_plugin_la_SOURCES = \ > + gst-plugin.cpp \ > + $(NULL) > + > +gst_plugin_la_CPPFLAGS = \ > + -I$(top_srcdir)/include \ > + $(SPICE_PROTOCOL_CFLAGS) \ > + $(GST_CFLAGS) \ > + $(NULL) > +endif > diff --git a/src/gst-plugin.cpp b/src/gst-plugin.cpp > new file mode 100644 > index 0000000..028033e > --- /dev/null > +++ b/src/gst-plugin.cpp > @@ -0,0 +1,466 @@ > +/* Plugin implementation for gstreamer encoder > + * > + * \copyright > + * Copyright 2018 Red Hat Inc. All rights reserved. > + */ > + > +#include <config.h> > +#include <cstring> > +#include <exception> > +#include <stdexcept> > +#include <sstream> > +#include <memory> > +#include <syslog.h> > +#include <unistd.h> > +#include <gst/gst.h> > +#include <gst/app/gstappsink.h> > + > +#define XLIB_CAPTURE 1 > +#if XLIB_CAPTURE > +#include <X11/Xlib.h> > +#include <gst/app/gstappsrc.h> > +#endif > + > +#include <spice-streaming-agent/plugin.hpp> > +#include <spice-streaming-agent/frame-capture.hpp> > + > +#define gst_syslog(priority, str, ...) syslog(priority, "Gstreamer plugin: " str, ## __VA_ARGS__); > + > +namespace spice { > +namespace streaming_agent { > +namespace gstreamer_plugin { > + > +struct GstreamerEncoderSettings > +{ > + int fps = 25; > + SpiceVideoCodecType codec = SPICE_VIDEO_CODEC_TYPE_H264; > + std::string encoder; > +}; > + > +template <typename T> > +struct GstObjectDeleter { > + void operator()(T* p) > + { > + gst_object_unref(p); > + } > +}; > + > +template <typename T> > +using gst_object_ptr = std::unique_ptr<T, GstObjectDeleter<T> >; > + > +struct GstCapsDeleter { > + void operator()(GstCaps* p) > + { > + gst_caps_unref(p); > + } > +}; > + > +using gst_caps_ptr = std::unique_ptr<GstCaps, GstCapsDeleter>; > + > +struct GstSampleDeleter { > + void operator()(GstSample* p) > + { > + gst_sample_unref(p); > + } > +}; > + > +using gst_sample_ptr = std::unique_ptr<GstSample, GstSampleDeleter>; > + > +class GstreamerFrameCapture final: public FrameCapture > +{ > +public: > + GstreamerFrameCapture(const GstreamerEncoderSettings &settings); > + ~GstreamerFrameCapture(); > + FrameInfo CaptureFrame() override; > + void Reset() override; > + SpiceVideoCodecType VideoCodecType() const override { > + return settings.codec; > + } > +private: > + void free_sample(); > + GstElement *get_encoder_plugin(const GstreamerEncoderSettings& settings, gst_caps_ptr &sink_caps); > + GstElement *get_capture_plugin(const GstreamerEncoderSettings& settings); > + void pipeline_init(const GstreamerEncoderSettings& settings); > +#if XLIB_CAPTURE > + void xlib_capture(); > + Display *dpy; > + XImage *image = nullptr; > +#endif > + gst_object_ptr<GstElement> pipeline, capture, sink; > + gst_sample_ptr sample; > + GstMapInfo map = {}; > + uint32_t last_width = ~0u, last_height = ~0u; > + uint32_t cur_width = 0, cur_height = 0; > + bool is_first = true; > + GstreamerEncoderSettings settings; // will be set by plugin settings > +}; > + > +class GstreamerPlugin final: public Plugin > +{ > +public: > + FrameCapture *CreateCapture() override; > + unsigned Rank() override; > + void ParseOptions(const ConfigureOption *options); > + SpiceVideoCodecType VideoCodecType() const override { > + return settings.codec; > + } > +private: > + GstreamerEncoderSettings settings; > +}; > + > +GstElement *GstreamerFrameCapture::get_capture_plugin(const GstreamerEncoderSettings& settings) > +{ > + GstElement *capture = nullptr; > + > +#if XLIB_CAPTURE > + capture = gst_element_factory_make("appsrc", "capture"); > +#else > + capture = gst_element_factory_make("ximagesrc", "capture"); > + g_object_set(capture, > + "use-damage", 0, > + nullptr); > + > +#endif > + return capture; > +} > + > +GstElement *GstreamerFrameCapture::get_encoder_plugin(const GstreamerEncoderSettings& settings, > + gst_caps_ptr &sink_caps) > +{ > + GList *encoders; > + GList *filtered; > + GstElement *encoder; > + GstElementFactory *factory = nullptr; > + std::stringstream caps_ss; > + > + switch (settings.codec) { > + case SPICE_VIDEO_CODEC_TYPE_H264: > + caps_ss << "video/x-h264" << ",stream-format=(string)byte-stream"; > + break; > + case SPICE_VIDEO_CODEC_TYPE_MJPEG: > + caps_ss << "image/jpeg"; > + break; > + case SPICE_VIDEO_CODEC_TYPE_VP8: > + caps_ss << "video/x-vp8"; > + break; > + case SPICE_VIDEO_CODEC_TYPE_VP9: > + caps_ss << "video/x-vp9"; > + break; > + case SPICE_VIDEO_CODEC_TYPE_H265: > + caps_ss << "video/x-h265"; > + break; > + default : /* Should not happen - just to avoid compiler's complaint */ > + return nullptr; > + } > + caps_ss << ",framerate=" << settings.fps << "/1"; > + > + encoders = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_VIDEO_ENCODER, GST_RANK_NONE); > + sink_caps.reset(gst_caps_from_string(caps_ss.str().c_str())); > + filtered = gst_element_factory_list_filter(encoders, sink_caps.get(), GST_PAD_SRC, false); > + if (filtered) { > + gst_syslog(LOG_NOTICE, "Looking for plugins which can produce a '%s' stream", caps_ss.str().c_str()); > + for (GList *l = filtered; l != nullptr; l = l->next) { > + if (!factory && !settings.encoder.compare(GST_ELEMENT_NAME(l->data))) { > + factory = (GstElementFactory*)l->data; > + } > + gst_syslog(LOG_NOTICE, "'%s' plugin is available", GST_ELEMENT_NAME(l->data)); > + } > + if (!factory && !settings.encoder.empty()) { > + gst_syslog(LOG_WARNING, "Specified encoder '%s' cannot produce '%s' stream, make sure matching gst.codec is specified", settings.encoder.c_str(), caps_ss.str().c_str()); > + } > + factory = factory ? factory : (GstElementFactory*)filtered->data; > + gst_syslog(LOG_NOTICE, "'%s' encoder plugin is used", GST_ELEMENT_NAME(factory)); > + > + } else { > + gst_syslog(LOG_ERR, "No suitable encoder was found for '%s'", caps_ss.str().c_str()); > + } > + > + encoder = factory ? gst_element_factory_create(factory, "encoder") : nullptr; > + if (encoder) { // Invalid properties will be ignored silently > + /* x264enc properties */ > + gst_util_set_object_arg(G_OBJECT(encoder), "tune", "zerolatency");// stillimage, fastdecode, zerolatency > + gst_util_set_object_arg(G_OBJECT(encoder), "bframes", "0"); > + gst_util_set_object_arg(G_OBJECT(encoder), "speed-preset", "1");// 1-ultrafast, 6-med, 9-veryslow > + } > + gst_plugin_feature_list_free(filtered); > + gst_plugin_feature_list_free(encoders); > + return encoder; > +} > + > +// Utility to add an element to a GstBin > +// This to check return value and update reference correctly > +void gst_bin_add(GstBin *bin, const gst_object_ptr<GstElement> &elem) > +{ > + if (::gst_bin_add(bin, elem.get())) { > + // ::gst_bin_add take ownership using floating references but > + // we still hold a reference in elem so update reference > + // according > + g_object_ref(elem.get()); > + } else { > + throw std::runtime_error("Gstreamer's element cannot be added"); > + } > +} > + > +void GstreamerFrameCapture::pipeline_init(const GstreamerEncoderSettings& settings) > +{ > + gboolean link; > + > + gst_object_ptr<GstElement> pipeline(gst_pipeline_new("pipeline")); > + if (!pipeline) { > + throw std::runtime_error("Gstreamer's pipeline element cannot be created"); > + } > + gst_object_ptr<GstElement> capture(get_capture_plugin(settings)); > + if (!capture) { > + throw std::runtime_error("Gstreamer's capture element cannot be created"); > + } > + gst_object_ptr<GstElement> convert(gst_element_factory_make("videoconvert", "convert")); > + if (!convert) { > + throw std::runtime_error("Gstreamer's 'videoconvert' element cannot be created"); > + } > + gst_caps_ptr sink_caps; > + gst_object_ptr<GstElement> encoder(get_encoder_plugin(settings, sink_caps)); > + if (!encoder) { > + throw std::runtime_error("Gstreamer's encoder element cannot be created"); > + } > + gst_object_ptr<GstElement> sink(gst_element_factory_make("appsink", "sink")); > + if (!sink) { > + throw std::runtime_error("Gstreamer's appsink element cannot be created"); > + } > + > + g_object_set(sink.get(), > + "sync", FALSE, > + "drop", TRUE, > + "max-buffers", 1, > + nullptr); > + > + GstBin *bin = GST_BIN(pipeline.get()); > + gst_bin_add(bin, capture); > + gst_bin_add(bin, convert); > + gst_bin_add(bin, encoder); > + gst_bin_add(bin, sink); > + > + //gst_caps_ptr caps(gst_caps_from_string("video/x-raw,format=(string)I420")); > + gst_caps_ptr caps(gst_caps_from_string("video/x-raw")); > + link = gst_element_link(capture.get(), convert.get()) && > + gst_element_link_filtered(convert.get(), encoder.get(), caps.get()) && > + gst_element_link_filtered(encoder.get(), sink.get(), sink_caps.get()); > + if (!link) { > + throw std::runtime_error("Linking gstreamer's elements failed"); > + } > + > +#if XLIB_CAPTURE > + dpy = XOpenDisplay(nullptr); > + if (!dpy) { > + throw std::runtime_error("Unable to initialize X11"); > + } > +#endif > + > + gst_element_set_state(pipeline.get(), GST_STATE_PLAYING); > + > +#if !XLIB_CAPTURE > + int sx, sy, ex, ey; > + g_object_get(capture.get(), > + "endx", &ex, > + "endy", &ey, > + "startx", &sx, > + "starty", &sy, > + nullptr); > + cur_width = ex - sx; > + cur_height = ey - sy; > + if (cur_width < 16 || cur_height < 16) { > + throw std::runtime_error("Invalid screen size"); > + } > + g_object_set(capture.get(), > + /* Some encoders cannot handle odd resolution make sure it's even number of pixels > + * pixel counting starts from zero here! > + */ > + "endx", cur_width - !(cur_width%2), > + "endy", cur_height - !(cur_height%2), > + "startx", 0, > + "starty", 0, > + nullptr); > +#endif > + > + this->sink.swap(sink); > + this->capture.swap(capture); > + this->pipeline.swap(pipeline); > +} > + > +GstreamerFrameCapture::GstreamerFrameCapture(const GstreamerEncoderSettings& settings): > + settings(settings) > +{ > + pipeline_init(settings); > +} > + > +void GstreamerFrameCapture::free_sample() > +{ > + if (sample) { > + gst_buffer_unmap(gst_sample_get_buffer(sample.get()), &map); > + sample.reset(); > + } > +#if XLIB_CAPTURE > + if(image) { > + image->f.destroy_image(image); > + image = nullptr; > + } > +#endif > +} > + > +GstreamerFrameCapture::~GstreamerFrameCapture() > +{ > + free_sample(); > + gst_element_set_state(pipeline.get(), GST_STATE_NULL); > +#if XLIB_CAPTURE > + XCloseDisplay(dpy); > +#endif > +} > + > +void GstreamerFrameCapture::Reset() > +{ > + //TODO > +} > + > +#if XLIB_CAPTURE > +void GstreamerFrameCapture::xlib_capture() > +{ > + int screen = XDefaultScreen(dpy); > + > + Window win = RootWindow(dpy, screen); > + XWindowAttributes win_info; > + XGetWindowAttributes(dpy, win, &win_info); > + > + /* Some encoders cannot handle odd resolution make sure it's even number of pixels */ > + cur_width = win_info.width - win_info.width%2; > + cur_height = win_info.height - win_info.height%2; > + > + if (cur_width != last_width || cur_height != last_height) { > + last_width = cur_width; > + last_height = cur_height; > + is_first = true; > + > + gst_app_src_end_of_stream(GST_APP_SRC(capture.get())); > + gst_element_set_state(pipeline.get(), GST_STATE_NULL);//maybe ximagesrc needs eos as well > + gst_element_set_state(pipeline.get(), GST_STATE_PLAYING); > + } > + > + image = XGetImage(dpy, win, 0, 0, > + cur_width, cur_height, AllPlanes, ZPixmap); > + if (!image) { > + throw std::runtime_error("Cannot capture from X"); > + } > + > + GstBuffer *buf; > + buf = gst_buffer_new_wrapped(image->data, image->height * image->bytes_per_line); > + if (!buf) { > + throw std::runtime_error("Failed to wrap image in gstreamer buffer"); > + } > + > + std::stringstream ss; > + > + ss << "video/x-raw,format=BGRx,width=" << image->width << ",height=" << image->height << ",framerate=" << settings.fps << "/1"; > + gst_caps_ptr caps(gst_caps_from_string(ss.str().c_str())); > + > + // Push sample > + gst_sample_ptr appsrc_sample(gst_sample_new(buf, caps.get(), nullptr, nullptr)); > + if (gst_app_src_push_sample(GST_APP_SRC(capture.get()), appsrc_sample.get()) != GST_FLOW_OK) { > + throw std::runtime_error("gstramer appsrc element cannot push sample"); > + } > +} > +#endif > + > +FrameInfo GstreamerFrameCapture::CaptureFrame() > +{ > + FrameInfo info; > + > + free_sample(); // free prev if exist > + > +#if XLIB_CAPTURE > + xlib_capture(); > +#endif > + info.size.width = cur_width; > + info.size.height = cur_height; > + info.stream_start = is_first; > + if (is_first) { > + is_first = false; > + } > + > + // Pull sample > + sample.reset(gst_app_sink_pull_sample(GST_APP_SINK(sink.get()))); // blocking > + > + if (sample) { // map after pipeline > + if (!gst_buffer_map(gst_sample_get_buffer(sample.get()), &map, GST_MAP_READ)) { > + free_sample(); > + throw std::runtime_error("Buffer mapping failed"); > + } > + > + info.buffer = map.data; > + info.buffer_size = map.size; > + } else { > + throw std::runtime_error("No sample- EOS or state change"); > + } > + > + return info; > +} > + > +FrameCapture *GstreamerPlugin::CreateCapture() > +{ > + return new GstreamerFrameCapture(settings); > +} > + > +unsigned GstreamerPlugin::Rank() > +{ > + return SoftwareMin; > +} > + > +void GstreamerPlugin::ParseOptions(const ConfigureOption *options) > +{ > + for (; options->name; ++options) { > + const std::string name = options->name; > + const std::string value = options->value; > + > + if (name == "framerate") { > + try { > + settings.fps = std::stoi(value); > + } catch (const std::exception &e) { > + throw std::runtime_error("Invalid value '" + value + "' for option 'framerate'."); > + } > + } else if (name == "gst.codec") { > + if (value == "h264") { > + settings.codec = SPICE_VIDEO_CODEC_TYPE_H264; > + } else if (value == "vp9") { > + settings.codec = SPICE_VIDEO_CODEC_TYPE_VP9; > + } else if (value == "vp8") { > + settings.codec = SPICE_VIDEO_CODEC_TYPE_VP8; > + } else if (value == "mjpeg") { > + settings.codec = SPICE_VIDEO_CODEC_TYPE_MJPEG; > + } else if (value == "h265") { > + settings.codec = SPICE_VIDEO_CODEC_TYPE_H265; > + } else { > + throw std::runtime_error("Invalid value '" + value + "' for option 'gst.codec'."); > + } > + } else if (name == "gst.encoder") { > + if (value.length() < 3) { > + gst_syslog(LOG_WARNING, "Encoder name length is invalid, will be ignored"); > + } else { > + settings.encoder = value; > + } > + } > + } > +} > + > +}}} //namespace spice::streaming_agent::gstreamer_plugin > + > +using namespace spice::streaming_agent::gstreamer_plugin; > + > +SPICE_STREAMING_AGENT_PLUGIN(agent) > +{ > + gst_init(nullptr, nullptr); > + > + std::unique_ptr<GstreamerPlugin> plugin(new GstreamerPlugin()); > + > + plugin->ParseOptions(agent->Options()); > + > + agent->Register(*plugin.release()); > + > + return true; > +} > -- Eduardo de Barros Lima (Etrunko) Software Engineer - RedHat etrunko@xxxxxxxxxx _______________________________________________ Spice-devel mailing list Spice-devel@xxxxxxxxxxxxxxxxxxxxx https://lists.freedesktop.org/mailman/listinfo/spice-devel