From e1f8f68330b3069158d215f2708260d777bc1c59 Mon Sep 17 00:00:00 2001 From: hack Date: Fri, 2 May 2025 18:02:34 -0500 Subject: [PATCH 01/37] use fbo rendering --- CMakeLists.txt | 2 + src/caps.c | 2 +- src/gstglbaseaudiovisualizer.c | 235 +++++--- src/gstglbaseaudiovisualizer.h | 18 +- src/gstpmaudiovisualizer.c | 998 +++++++++++++++++++++++++++++++++ src/gstpmaudiovisualizer.h | 97 ++++ src/plugin.c | 203 +++++-- src/plugin.h | 7 +- src/projectm.c | 18 +- 9 files changed, 1455 insertions(+), 125 deletions(-) create mode 100644 src/gstpmaudiovisualizer.c create mode 100644 src/gstpmaudiovisualizer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e41f87f..3038cab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,8 @@ add_library(gstprojectm SHARED src/projectm.c src/gstglbaseaudiovisualizer.h src/gstglbaseaudiovisualizer.c + src/gstpmaudiovisualizer.h + src/gstpmaudiovisualizer.c ) target_include_directories(gstprojectm diff --git a/src/caps.c b/src/caps.c index 09b5add..c7a1941 100644 --- a/src/caps.c +++ b/src/caps.c @@ -39,7 +39,7 @@ const gchar *get_video_src_cap(unsigned int type) { switch (type) { case 0: - format = GST_VIDEO_CAPS_MAKE("video/x-raw, format = (string) { ABGR }, " + format = GST_VIDEO_CAPS_MAKE("video/x-raw, format = (string) { RGBA }, " "framerate=(fraction)[0/1,MAX]"); break; default: diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index e1f786a..097fdd6 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -35,16 +35,17 @@ #endif #include "gstglbaseaudiovisualizer.h" +#include "gstpmaudiovisualizer.h" #include /** * SECTION:GstGLBaseAudioVisualizer - * @short_description: #GstAudioVisualizer subclass for injecting OpenGL + * @short_description: #GstPMAudioVisualizer subclass for injecting OpenGL * resources in a pipeline * @title: GstGLBaseAudioVisualizer - * @see_also: #GstAudioVisualizer + * @see_also: #GstPMAudioVisualizer * - * Wrapper for GstAudioVisualizer for handling OpenGL contexts. + * Wrapper for GstPMAudioVisualizer for handling OpenGL contexts. * * #GstGLBaseAudioVisualizer handles the nitty gritty details of retrieving an * OpenGL context. It also provides `gl_start()` and `gl_stop()` virtual methods @@ -58,8 +59,11 @@ GST_DEBUG_CATEGORY_STATIC(GST_CAT_DEFAULT); struct _GstGLBaseAudioVisualizerPrivate { GstGLContext *other_context; + GstGLMemory *out_tex; + GstBuffer *in_audio; - gint64 n_frames; /* total frames sent */ + gint64 timestamp_offset; /* base offset */ + gint64 n_frames; /* total frames sent */ gboolean gl_result; gboolean gl_started; @@ -67,12 +71,12 @@ struct _GstGLBaseAudioVisualizerPrivate { }; /* Properties */ -enum { PROP_0 }; +enum { PROP_0, PROP_TIMESTAMP_OFFSET }; #define gst_gl_base_audio_visualizer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE( GstGLBaseAudioVisualizer, gst_gl_base_audio_visualizer, - GST_TYPE_AUDIO_VISUALIZER, + GST_TYPE_PM_AUDIO_VISUALIZER, G_ADD_PRIVATE(GstGLBaseAudioVisualizer) GST_DEBUG_CATEGORY_INIT(gst_gl_base_audio_visualizer_debug, "glbaseaudiovisualizer", 0, @@ -94,13 +98,13 @@ static GstStateChangeReturn gst_gl_base_audio_visualizer_change_state(GstElement *element, GstStateChange transition); -static gboolean gst_gl_base_audio_visualizer_render(GstAudioVisualizer *bscope, - GstBuffer *audio, - GstVideoFrame *video); +static gboolean +gst_gl_base_audio_visualizer_render(GstPMAudioVisualizer *bscope, + GstBuffer *audio, GstVideoFrame *video); static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav); static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav); static gboolean -gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, +gst_gl_base_audio_visualizer_decide_allocation(GstPMAudioVisualizer *gstav, GstQuery *query); static gboolean @@ -109,18 +113,24 @@ static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav); static void gst_gl_base_audio_visualizer_default_gl_stop(GstGLBaseAudioVisualizer *glav); -static gboolean gst_gl_base_audio_visualizer_default_gl_render( - GstGLBaseAudioVisualizer *glav, GstBuffer *audio, GstVideoFrame *video); +static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( + GstGLBaseAudioVisualizer *glav, GstBuffer *in_audio, GstGLMemory *mem); static gboolean gst_gl_base_audio_visualizer_find_gl_context_unlocked( GstGLBaseAudioVisualizer *glav); -static gboolean gst_gl_base_audio_visualizer_setup(GstAudioVisualizer *gstav); +static gboolean gst_gl_base_audio_visualizer_setup(GstPMAudioVisualizer *gstav); + +static GstFlowReturn gst_gl_base_audio_visualizer_default_prepare_output_buffer( + GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf); + +static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( + GstPMAudioVisualizer *scope, GstBuffer **outbuf); static void gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); - GstAudioVisualizerClass *gstav_class = GST_AUDIO_VISUALIZER_CLASS(klass); + GstPMAudioVisualizerClass *gstav_class = GST_PM_AUDIO_VISUALIZER_CLASS(klass); GstElementClass *element_class = GST_ELEMENT_CLASS(klass); gobject_class->finalize = gst_gl_base_audio_visualizer_finalize; @@ -138,22 +148,30 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { gstav_class->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_setup); gstav_class->render = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_render); + gstav_class->prepare_output_buffer = GST_DEBUG_FUNCPTR( + gst_gl_base_audio_visualizer_parent_prepare_output_buffer); klass->supported_gl_api = GST_GL_API_ANY; klass->gl_start = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_start); klass->gl_stop = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_stop); - klass->gl_render = - GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_render); - klass->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_setup); + klass->setup = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_setup); + klass->fill_gl_memory = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_fill_gl_memory); + klass->prepare_output_buffer = GST_DEBUG_FUNCPTR( + gst_gl_base_audio_visualizer_default_prepare_output_buffer); } static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv = gst_gl_base_audio_visualizer_get_instance_private(glav); glav->priv->gl_started = FALSE; glav->priv->gl_result = TRUE; + glav->priv->in_audio = NULL; + glav->priv->out_tex = NULL; glav->context = NULL; + glav->priv->timestamp_offset = 0; g_rec_mutex_init(&glav->priv->context_lock); gst_gl_base_audio_visualizer_start(glav); } @@ -174,6 +192,11 @@ static void gst_gl_base_audio_visualizer_set_property(GObject *object, GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); switch (prop_id) { + + case PROP_TIMESTAMP_OFFSET: + glav->priv->timestamp_offset = g_value_get_int64(value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -187,6 +210,11 @@ static void gst_gl_base_audio_visualizer_get_property(GObject *object, GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); switch (prop_id) { + + case PROP_TIMESTAMP_OFFSET: + g_value_set_int64(value, glav->priv->timestamp_offset); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -239,6 +267,7 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(data); GstGLBaseAudioVisualizerClass *glav_class = GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(glav); + GstPMAudioVisualizer *gstav = GST_PM_AUDIO_VISUALIZER(glav); GST_INFO_OBJECT(glav, "starting"); gst_gl_insert_debug_marker(glav->context, "starting element %s", @@ -266,81 +295,155 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, glav->priv->gl_started = FALSE; } -static gboolean gst_gl_base_audio_visualizer_setup(GstAudioVisualizer *gstav) { - GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); - GstGLBaseAudioVisualizerClass *glav_class = - GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(gstav); +static GstFlowReturn gst_gl_base_audio_visualizer_default_prepare_output_buffer( + GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf) { + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(scope); + return gst_pm_audio_visualizer_prepare_output_buffer(pmav, outbuf); +} - // cascade setup to the derived plugin after gl initialization has been - // completed - return glav_class->setup(glav); +static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( + GstPMAudioVisualizer *scope, GstBuffer **outbuf) { + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(scope); + GstGLBaseAudioVisualizerClass *klass = + GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(glav); + return klass->prepare_output_buffer(glav, outbuf); } -static gboolean gst_gl_base_audio_visualizer_default_gl_render( - GstGLBaseAudioVisualizer *glav, GstBuffer *audio, GstVideoFrame *video) { +static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( + GstGLBaseAudioVisualizer *glav, GstBuffer *in_audio, GstGLMemory *mem) { return TRUE; } -typedef struct { - GstGLBaseAudioVisualizer *glav; - GstBuffer *in_audio; - GstVideoFrame *out_video; -} GstGLRenderCallbackParams; - -static void -gst_gl_base_audio_visualizer_gl_thread_render_callback(gpointer params) { - GstGLRenderCallbackParams *cb_params = (GstGLRenderCallbackParams *)params; +static void _fill_gl(GstGLContext *context, GstGLBaseAudioVisualizer *glav) { GstGLBaseAudioVisualizerClass *klass = - GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(cb_params->glav); + GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(glav); + GST_TRACE_OBJECT(glav, "filling gl memory %p", glav->priv->out_tex); // inside gl thread: call virtual render function with audio and video - cb_params->glav->priv->gl_result = klass->gl_render( - cb_params->glav, cb_params->in_audio, cb_params->out_video); + glav->priv->gl_result = + klass->fill_gl_memory(glav, glav->priv->in_audio, glav->priv->out_tex); } -static gboolean gst_gl_base_audio_visualizer_render(GstAudioVisualizer *bscope, - GstBuffer *audio, - GstVideoFrame *video) { - GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(bscope); - GstGLRenderCallbackParams cb_params; - GstGLWindow *window; +static GstFlowReturn +gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, + GstGLBaseAudioVisualizer *glav, + GstBuffer *audio, GstVideoFrame *video) { + GstClockTime next_time; + GstGLSyncMeta *sync_meta; g_rec_mutex_lock(&glav->priv->context_lock); + if (G_UNLIKELY(!glav->context)) + goto not_negotiated; + + /* 0 framerate and we are at the second frame, eos */ + if (G_UNLIKELY(GST_VIDEO_INFO_FPS_N(&bscope->vinfo) == 0 && + glav->priv->n_frames == 1)) + goto eos; + + // there is an issue here: the video buffer has already been mapped without + // GST_MAP_GL flag + + /* + GstBuffer *buffer; + + // Allocate a buffer of specific size + buffer = gst_gl_buffer_new_wrapped (g_malloc0 (bscope->vinfo.size), + bscope->vinfo.size); + + // Map the buffer to set its memory data + GstVideoFrame map; - // wrap params into cb_params struct to pass them to the GL window/thread via - // userdata pointer - cb_params.glav = glav; - cb_params.in_audio = audio; - cb_params.out_video = video; + //gst_video_frame_unmap (video); - window = gst_gl_context_get_window(glav->context); + if (!gst_video_frame_map (&map, &bscope->vinfo, buffer, + GST_MAP_WRITE | GST_MAP_GL)) { + return GST_FLOW_NOT_NEGOTIATED; + }*/ - // dispatch render call through the gl thread - // call is blocking, accessing audio and video params from gl thread *should* - // be safe - gst_gl_window_send_message( - window, - GST_GL_WINDOW_CB(gst_gl_base_audio_visualizer_gl_thread_render_callback), - &cb_params); + glav->priv->out_tex = (GstGLMemory *)video->map[0].memory; + glav->priv->in_audio = audio; - gst_object_unref(window); + gst_gl_context_thread_add(glav->context, (GstGLContextThreadFunc)_fill_gl, + glav); + + glav->priv->out_tex = NULL; + glav->priv->in_audio = NULL; + + if (!glav->priv->gl_result) + goto gl_error; + + GstBuffer *buffer = video->buffer; + + sync_meta = gst_buffer_get_gl_sync_meta(buffer); + if (sync_meta) + gst_gl_sync_meta_set_sync_point(sync_meta, glav->context); g_rec_mutex_unlock(&glav->priv->context_lock); - if (glav->priv->gl_result) { - glav->priv->n_frames++; + GST_BUFFER_TIMESTAMP(buffer) = + glav->priv->timestamp_offset + glav->running_time; + GST_BUFFER_OFFSET(buffer) = glav->priv->n_frames; + glav->priv->n_frames++; + GST_BUFFER_OFFSET_END(buffer) = glav->priv->n_frames; + if (bscope->vinfo.fps_n) { + next_time = + gst_util_uint64_scale_int(glav->priv->n_frames * GST_SECOND, + bscope->vinfo.fps_d, bscope->vinfo.fps_n); + GST_BUFFER_DURATION(buffer) = next_time - glav->running_time; } else { - // gl error - GST_ELEMENT_ERROR(glav, RESOURCE, NOT_FOUND, - (("failed to render audio visualizer")), - (("A GL error occurred"))); + next_time = glav->priv->timestamp_offset; + /* NONE means forever */ + GST_BUFFER_DURATION(buffer) = GST_CLOCK_TIME_NONE; } + glav->running_time = next_time; + + return GST_FLOW_OK; + +gl_error: { + g_rec_mutex_unlock(&glav->priv->context_lock); + GST_ELEMENT_ERROR(glav, RESOURCE, NOT_FOUND, (("failed to draw pattern")), + (("A GL error occurred"))); + return GST_FLOW_NOT_NEGOTIATED; +} +not_negotiated: { + g_rec_mutex_unlock(&glav->priv->context_lock); + GST_ELEMENT_ERROR(glav, CORE, NEGOTIATION, (NULL), + (("format wasn't negotiated before get function"))); + return GST_FLOW_NOT_NEGOTIATED; +} +eos: { + g_rec_mutex_unlock(&glav->priv->context_lock); + GST_DEBUG_OBJECT(glav, "eos: 0 framerate, frame %d", + (gint)glav->priv->n_frames); + return GST_FLOW_EOS; +} +} + +static gboolean +gst_gl_base_audio_visualizer_setup(GstPMAudioVisualizer *gstav) { + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); + GstGLBaseAudioVisualizerClass *glav_class = + GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(gstav); + + // cascade setup to the derived plugin after gl initialization has been + // completed + return glav_class->setup(glav); +} + +static gboolean +gst_gl_base_audio_visualizer_render(GstPMAudioVisualizer *bscope, + GstBuffer *audio, GstVideoFrame *video) { + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(bscope); + + gst_gl_base_audio_visualizer_fill(bscope, glav, audio, video); + return glav->priv->gl_result; } static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav) { glav->priv->n_frames = 0; + glav->running_time = 0; } static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav) { @@ -494,7 +597,7 @@ error: { } static gboolean -gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, +gst_gl_base_audio_visualizer_decide_allocation(GstPMAudioVisualizer *gstav, GstQuery *query) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); GstGLContext *context; @@ -529,7 +632,7 @@ gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, } if (!pool || !GST_IS_GL_BUFFER_POOL(pool)) { - /* can't use this pool */ + // can't use this pool if (pool) gst_object_unref(pool); pool = gst_gl_buffer_pool_new(context); diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index a48781b..546eb6f 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -32,8 +32,8 @@ #ifndef __GST_GL_BASE_AUDIO_VISUALIZER_H__ #define __GST_GL_BASE_AUDIO_VISUALIZER_H__ +#include "gstpmaudiovisualizer.h" #include -#include #include #include @@ -72,12 +72,15 @@ GType gst_gl_base_audio_visualizer_get_type(void); * The parent instance type of a base GL Audio Visualizer. */ struct _GstGLBaseAudioVisualizer { - GstAudioVisualizer parent; + GstPMAudioVisualizer parent; /*< public >*/ GstGLDisplay *display; GstGLContext *context; + /* total running time */ + GstClockTime running_time; + /*< private >*/ gpointer _padding[GST_PADDING]; @@ -91,21 +94,24 @@ struct _GstGLBaseAudioVisualizer { * @gl_stop: called in the GL thread to clean up the element GL state. * @gl_render: called in the GL thread to fill the current video texture. * @setup: called when the format changes (delegate from - * GstAudioVisualizer.setup) + * GstPMAudioVisualizer.setup) * * The base class for OpenGL based audio visualizers. * */ struct _GstGLBaseAudioVisualizerClass { - GstAudioVisualizerClass parent_class; + GstPMAudioVisualizerClass parent_class; /*< public >*/ GstGLAPI supported_gl_api; gboolean (*gl_start)(GstGLBaseAudioVisualizer *glav); void (*gl_stop)(GstGLBaseAudioVisualizer *glav); - gboolean (*gl_render)(GstGLBaseAudioVisualizer *glav, GstBuffer *audio, - GstVideoFrame *video); gboolean (*setup)(GstGLBaseAudioVisualizer *glav); + gboolean (*fill_gl_memory)(GstGLBaseAudioVisualizer *glav, + GstBuffer *in_audio, GstGLMemory *mem); + GstFlowReturn (*prepare_output_buffer)(GstGLBaseAudioVisualizer *glav, + GstBuffer **outbuf); + /*< private >*/ gpointer _padding[GST_PADDING]; }; diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c new file mode 100644 index 0000000..1aba979 --- /dev/null +++ b/src/gstpmaudiovisualizer.c @@ -0,0 +1,998 @@ +/* GStreamer + * Copyright (C) <2011> Stefan Kost + * Copyright (C) <2015> Luis de Bethencourt + * + * gstaudiovisualizer.h: base class for audio visualisation elements + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +/** + * SECTION:gstaudiovisualizer + * @title: GstPMAudioVisualizer + * @short_description: Base class for visualizers. + * + * A baseclass for scopes (visualizers). It takes care of re-fitting the + * audio-rate to video-rate and handles renegotiation (downstream video size + * changes). + * + * It also provides several background shading effects. These effects are + * applied to a previous picture before the `render()` implementation can draw a + * new frame. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include +#include +#include +#include + +#include "gstpmaudiovisualizer.h" +#include +#include + +GST_DEBUG_CATEGORY_STATIC(audio_visualizer_debug); +#define GST_CAT_DEFAULT (audio_visualizer_debug) + +#define DEFAULT_SHADER GST_AUDIO_VISUALIZER_SHADER_FADE +#define DEFAULT_SHADE_AMOUNT 0x000a0a0a + +enum { PROP_0 }; + +static GstBaseTransformClass *parent_class = NULL; +static gint private_offset = 0; + +static void gst_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass); +static void gst_audio_visualizer_init(GstPMAudioVisualizer *scope, + GstPMAudioVisualizerClass *g_class); +static void gst_audio_visualizer_set_property(GObject *object, guint prop_id, + const GValue *value, + GParamSpec *pspec); +static void gst_audio_visualizer_get_property(GObject *object, guint prop_id, + GValue *value, GParamSpec *pspec); +static void gst_audio_visualizer_dispose(GObject *object); + +static gboolean gst_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope); +static gboolean gst_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps); +static gboolean gst_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps); + +static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, + GstBuffer *buffer); + +static gboolean gst_audio_visualizer_src_event(GstPad *pad, GstObject *parent, + GstEvent *event); +static gboolean gst_audio_visualizer_sink_event(GstPad *pad, GstObject *parent, + GstEvent *event); + +static gboolean gst_audio_visualizer_src_query(GstPad *pad, GstObject *parent, + GstQuery *query); + +static GstStateChangeReturn +gst_audio_visualizer_change_state(GstElement *element, + GstStateChange transition); + +static gboolean gst_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, + GstCaps *outcaps); + +static gboolean default_decide_allocation(GstPMAudioVisualizer *scope, + GstQuery *query); + +struct _GstPMAudioVisualizerPrivate { + gboolean negotiated; + + GstBufferPool *pool; + gboolean pool_active; + GstAllocator *allocator; + GstAllocationParams params; + GstQuery *query; + + /* pads */ + GstPad *srcpad, *sinkpad; + + GstAdapter *adapter; + + GstBuffer *inbuf; + GstBuffer *tempbuf; + GstVideoFrame tempframe; + + guint spf; /* samples per video frame */ + guint64 frame_duration; + + /* QoS stuff */ /* with LOCK */ + gdouble proportion; + GstClockTime earliest_time; + + guint dropped; /* frames dropped / not dropped */ + guint processed; + + /* configuration mutex */ + GMutex config_lock; + + GstSegment segment; +}; + +/* base class */ + +GType gst_pm_audio_visualizer_get_type(void) { + static gsize audio_visualizer_type = 0; + + if (g_once_init_enter(&audio_visualizer_type)) { + static const GTypeInfo audio_visualizer_info = { + sizeof(GstPMAudioVisualizerClass), + NULL, + NULL, + (GClassInitFunc)gst_audio_visualizer_class_init, + NULL, + NULL, + sizeof(GstPMAudioVisualizer), + 0, + (GInstanceInitFunc)gst_audio_visualizer_init, + }; + GType _type; + + /* TODO: rename when exporting it as a library */ + _type = + g_type_register_static(GST_TYPE_ELEMENT, "GstPMAudioVisualizer", + &audio_visualizer_info, G_TYPE_FLAG_ABSTRACT); + + private_offset = + g_type_add_instance_private(_type, sizeof(GstPMAudioVisualizerPrivate)); + + g_once_init_leave(&audio_visualizer_type, _type); + } + return (GType)audio_visualizer_type; +} + +static inline GstPMAudioVisualizerPrivate * +gst_audio_visualizer_get_instance_private(GstPMAudioVisualizer *self) { + return (G_STRUCT_MEMBER_P(self, private_offset)); +} + +static void gst_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass) { + GObjectClass *gobject_class = (GObjectClass *)klass; + GstElementClass *element_class = (GstElementClass *)klass; + + if (private_offset != 0) + g_type_class_adjust_private_offset(klass, &private_offset); + + parent_class = g_type_class_peek_parent(klass); + + GST_DEBUG_CATEGORY_INIT(audio_visualizer_debug, + "baseaudiovisualizer-libvisual", 0, + "scope audio visualisation base class"); + + gobject_class->set_property = gst_audio_visualizer_set_property; + gobject_class->get_property = gst_audio_visualizer_get_property; + gobject_class->dispose = gst_audio_visualizer_dispose; + + element_class->change_state = + GST_DEBUG_FUNCPTR(gst_audio_visualizer_change_state); + + klass->decide_allocation = GST_DEBUG_FUNCPTR(default_decide_allocation); + klass->prepare_output_buffer = + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_prepare_output_buffer); +} + +static void gst_audio_visualizer_init(GstPMAudioVisualizer *scope, + GstPMAudioVisualizerClass *g_class) { + GstPadTemplate *pad_template; + + scope->priv = gst_audio_visualizer_get_instance_private(scope); + + /* create the sink and src pads */ + pad_template = + gst_element_class_get_pad_template(GST_ELEMENT_CLASS(g_class), "sink"); + g_return_if_fail(pad_template != NULL); + scope->priv->sinkpad = gst_pad_new_from_template(pad_template, "sink"); + gst_pad_set_chain_function(scope->priv->sinkpad, + GST_DEBUG_FUNCPTR(gst_audio_visualizer_chain)); + gst_pad_set_event_function( + scope->priv->sinkpad, GST_DEBUG_FUNCPTR(gst_audio_visualizer_sink_event)); + gst_element_add_pad(GST_ELEMENT(scope), scope->priv->sinkpad); + + pad_template = + gst_element_class_get_pad_template(GST_ELEMENT_CLASS(g_class), "src"); + g_return_if_fail(pad_template != NULL); + scope->priv->srcpad = gst_pad_new_from_template(pad_template, "src"); + gst_pad_set_event_function(scope->priv->srcpad, + GST_DEBUG_FUNCPTR(gst_audio_visualizer_src_event)); + gst_pad_set_query_function(scope->priv->srcpad, + GST_DEBUG_FUNCPTR(gst_audio_visualizer_src_query)); + gst_element_add_pad(GST_ELEMENT(scope), scope->priv->srcpad); + + scope->priv->adapter = gst_adapter_new(); + scope->priv->inbuf = gst_buffer_new(); + + /* properties */ + + /* reset the initial video state */ + gst_video_info_init(&scope->vinfo); + scope->priv->frame_duration = GST_CLOCK_TIME_NONE; + + /* reset the initial state */ + gst_audio_info_init(&scope->ainfo); + gst_video_info_init(&scope->vinfo); + + g_mutex_init(&scope->priv->config_lock); +} + +static void gst_audio_visualizer_set_property(GObject *object, guint prop_id, + const GValue *value, + GParamSpec *pspec) { + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_audio_visualizer_get_property(GObject *object, guint prop_id, + GValue *value, + GParamSpec *pspec) { + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_audio_visualizer_dispose(GObject *object) { + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + + if (scope->priv->adapter) { + g_object_unref(scope->priv->adapter); + scope->priv->adapter = NULL; + } + if (scope->priv->inbuf) { + gst_buffer_unref(scope->priv->inbuf); + scope->priv->inbuf = NULL; + } + if (scope->priv->tempbuf) { + gst_video_frame_unmap(&scope->priv->tempframe); + gst_buffer_unref(scope->priv->tempbuf); + scope->priv->tempbuf = NULL; + } + if (scope->priv->config_lock.p) { + g_mutex_clear(&scope->priv->config_lock); + scope->priv->config_lock.p = NULL; + } + G_OBJECT_CLASS(parent_class)->dispose(object); +} + +static void gst_audio_visualizer_reset(GstPMAudioVisualizer *scope) { + gst_adapter_clear(scope->priv->adapter); + gst_segment_init(&scope->priv->segment, GST_FORMAT_UNDEFINED); + + GST_OBJECT_LOCK(scope); + scope->priv->proportion = 1.0; + scope->priv->earliest_time = -1; + scope->priv->dropped = 0; + scope->priv->processed = 0; + GST_OBJECT_UNLOCK(scope); +} + +static gboolean gst_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps) { + GstAudioInfo info; + + if (!gst_audio_info_from_caps(&info, caps)) + goto wrong_caps; + + scope->ainfo = info; + + GST_DEBUG_OBJECT(scope, "audio: channels %d, rate %d", + GST_AUDIO_INFO_CHANNELS(&info), GST_AUDIO_INFO_RATE(&info)); + + if (!gst_audio_visualizer_src_negotiate(scope)) { + goto not_negotiated; + } + + return TRUE; + + /* Errors */ +wrong_caps: { + GST_WARNING_OBJECT(scope, "could not parse caps"); + return FALSE; +} +not_negotiated: { + GST_WARNING_OBJECT(scope, "failed to negotiate"); + return FALSE; +} +} + +static gboolean gst_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps) { + GstVideoInfo info; + GstPMAudioVisualizerClass *klass; + gboolean res; + + if (!gst_video_info_from_caps(&info, caps)) + goto wrong_caps; + + klass = GST_PM_AUDIO_VISUALIZER_CLASS(G_OBJECT_GET_CLASS(scope)); + + scope->vinfo = info; + + scope->priv->frame_duration = gst_util_uint64_scale_int( + GST_SECOND, GST_VIDEO_INFO_FPS_D(&info), GST_VIDEO_INFO_FPS_N(&info)); + scope->priv->spf = gst_util_uint64_scale_int( + GST_AUDIO_INFO_RATE(&scope->ainfo), GST_VIDEO_INFO_FPS_D(&info), + GST_VIDEO_INFO_FPS_N(&info)); + scope->req_spf = scope->priv->spf; + + if (scope->priv->tempbuf) { + gst_video_frame_unmap(&scope->priv->tempframe); + gst_buffer_unref(scope->priv->tempbuf); + } + scope->priv->tempbuf = + gst_buffer_new_wrapped(g_malloc0(scope->vinfo.size), scope->vinfo.size); + gst_video_frame_map(&scope->priv->tempframe, &scope->vinfo, + scope->priv->tempbuf, GST_MAP_READWRITE); + + if (klass->setup && !klass->setup(scope)) + goto setup_failed; + + GST_DEBUG_OBJECT(scope, "video: dimension %dx%d, framerate %d/%d", + GST_VIDEO_INFO_WIDTH(&info), GST_VIDEO_INFO_HEIGHT(&info), + GST_VIDEO_INFO_FPS_N(&info), GST_VIDEO_INFO_FPS_D(&info)); + GST_DEBUG_OBJECT(scope, "blocks: spf %u, req_spf %u", scope->priv->spf, + scope->req_spf); + + gst_pad_set_caps(scope->priv->srcpad, caps); + + /* find a pool for the negotiated caps now */ + res = gst_audio_visualizer_do_bufferpool(scope, caps); + gst_caps_unref(caps); + + return res; + + /* ERRORS */ +wrong_caps: { + gst_caps_unref(caps); + GST_DEBUG_OBJECT(scope, "error parsing caps"); + return FALSE; +} + +setup_failed: { + GST_WARNING_OBJECT(scope, "failed to set up"); + return FALSE; +} +} + +static gboolean +gst_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope) { + GstCaps *othercaps, *target; + GstStructure *structure; + GstCaps *templ; + gboolean ret; + + templ = gst_pad_get_pad_template_caps(scope->priv->srcpad); + + GST_DEBUG_OBJECT(scope, "performing negotiation"); + + /* see what the peer can do */ + othercaps = gst_pad_peer_query_caps(scope->priv->srcpad, NULL); + if (othercaps) { + target = gst_caps_intersect(othercaps, templ); + gst_caps_unref(othercaps); + gst_caps_unref(templ); + + if (gst_caps_is_empty(target)) + goto no_format; + + target = gst_caps_truncate(target); + } else { + target = templ; + } + + target = gst_caps_make_writable(target); + structure = gst_caps_get_structure(target, 0); + gst_structure_fixate_field_nearest_int(structure, "width", 320); + gst_structure_fixate_field_nearest_int(structure, "height", 200); + gst_structure_fixate_field_nearest_fraction(structure, "framerate", 25, 1); + if (gst_structure_has_field(structure, "pixel-aspect-ratio")) + gst_structure_fixate_field_nearest_fraction(structure, "pixel-aspect-ratio", + 1, 1); + + target = gst_caps_fixate(target); + + GST_DEBUG_OBJECT(scope, "final caps are %" GST_PTR_FORMAT, target); + + ret = gst_audio_visualizer_src_setcaps(scope, target); + + return ret; + +no_format: { + gst_caps_unref(target); + return FALSE; +} +} + +/* takes ownership of the pool, allocator and query */ +static gboolean gst_audio_visualizer_set_allocation( + GstPMAudioVisualizer *scope, GstBufferPool *pool, GstAllocator *allocator, + const GstAllocationParams *params, GstQuery *query) { + GstAllocator *oldalloc; + GstBufferPool *oldpool; + GstQuery *oldquery; + GstPMAudioVisualizerPrivate *priv = scope->priv; + + GST_OBJECT_LOCK(scope); + oldpool = priv->pool; + priv->pool = pool; + priv->pool_active = FALSE; + + oldalloc = priv->allocator; + priv->allocator = allocator; + + oldquery = priv->query; + priv->query = query; + + if (params) + priv->params = *params; + else + gst_allocation_params_init(&priv->params); + GST_OBJECT_UNLOCK(scope); + + if (oldpool) { + GST_DEBUG_OBJECT(scope, "deactivating old pool %p", oldpool); + gst_buffer_pool_set_active(oldpool, FALSE); + gst_object_unref(oldpool); + } + if (oldalloc) { + gst_object_unref(oldalloc); + } + if (oldquery) { + gst_query_unref(oldquery); + } + return TRUE; +} + +static gboolean gst_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, + GstCaps *outcaps) { + GstQuery *query; + gboolean result = TRUE; + GstBufferPool *pool = NULL; + GstPMAudioVisualizerClass *klass; + GstAllocator *allocator; + GstAllocationParams params; + + /* not passthrough, we need to allocate */ + /* find a pool for the negotiated caps now */ + GST_DEBUG_OBJECT(scope, "doing allocation query"); + query = gst_query_new_allocation(outcaps, TRUE); + + if (!gst_pad_peer_query(scope->priv->srcpad, query)) { + /* not a problem, we use the query defaults */ + GST_DEBUG_OBJECT(scope, "allocation query failed"); + } + + klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); + + GST_DEBUG_OBJECT(scope, "calling decide_allocation"); + g_assert(klass->decide_allocation != NULL); + result = klass->decide_allocation(scope, query); + + GST_DEBUG_OBJECT(scope, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, + query); + + if (!result) + goto no_decide_allocation; + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params(query) > 0) { + gst_query_parse_nth_allocation_param(query, 0, &allocator, ¶ms); + } else { + allocator = NULL; + gst_allocation_params_init(¶ms); + } + + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + + /* now store */ + result = gst_audio_visualizer_set_allocation(scope, pool, allocator, ¶ms, + query); + + return result; + + /* Errors */ +no_decide_allocation: { + GST_WARNING_OBJECT(scope, "Subclass failed to decide allocation"); + gst_query_unref(query); + + return result; +} +} + +static gboolean default_decide_allocation(GstPMAudioVisualizer *scope, + GstQuery *query) { + GstCaps *outcaps; + GstBufferPool *pool; + guint size, min, max; + GstAllocator *allocator; + GstAllocationParams params; + GstStructure *config; + gboolean update_allocator; + gboolean update_pool; + + gst_query_parse_allocation(query, &outcaps, NULL); + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params(query) > 0) { + /* try the allocator */ + gst_query_parse_nth_allocation_param(query, 0, &allocator, ¶ms); + update_allocator = TRUE; + } else { + allocator = NULL; + gst_allocation_params_init(¶ms); + update_allocator = FALSE; + } + + if (gst_query_get_n_allocation_pools(query) > 0) { + gst_query_parse_nth_allocation_pool(query, 0, &pool, &size, &min, &max); + update_pool = TRUE; + } else { + pool = NULL; + size = GST_VIDEO_INFO_SIZE(&scope->vinfo); + min = max = 0; + update_pool = FALSE; + } + + if (pool == NULL) { + /* we did not get a pool, make one ourselves then */ + pool = gst_video_buffer_pool_new(); + } + + config = gst_buffer_pool_get_config(pool); + gst_buffer_pool_config_set_params(config, outcaps, size, min, max); + gst_buffer_pool_config_set_allocator(config, allocator, ¶ms); + gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_set_config(pool, config); + + if (update_allocator) + gst_query_set_nth_allocation_param(query, 0, allocator, ¶ms); + else + gst_query_add_allocation_param(query, allocator, ¶ms); + + if (allocator) + gst_object_unref(allocator); + + if (update_pool) + gst_query_set_nth_allocation_pool(query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool(query, pool, size, min, max); + + if (pool) + gst_object_unref(pool); + + return TRUE; +} + +GstFlowReturn +gst_pm_audio_visualizer_prepare_output_buffer(GstPMAudioVisualizer *scope, + GstBuffer **outbuf) { + GstPMAudioVisualizerPrivate *priv; + + priv = scope->priv; + + g_assert(priv->pool != NULL); + + /* we can't reuse the input buffer */ + if (!priv->pool_active) { + GST_DEBUG_OBJECT(scope, "setting pool %p active", priv->pool); + if (!gst_buffer_pool_set_active(priv->pool, TRUE)) + goto activate_failed; + priv->pool_active = TRUE; + } + GST_DEBUG_OBJECT(scope, "using pool alloc"); + + return gst_buffer_pool_acquire_buffer(priv->pool, outbuf, NULL); + + /* ERRORS */ +activate_failed: { + GST_ELEMENT_ERROR(scope, RESOURCE, SETTINGS, + ("failed to activate bufferpool"), + ("failed to activate bufferpool")); + return GST_FLOW_ERROR; +} +} + +static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, + GstBuffer *buffer) { + GstFlowReturn ret = GST_FLOW_OK; + GstPMAudioVisualizer *scope; + GstPMAudioVisualizerClass *klass; + GstBuffer *inbuf; + guint64 dist, ts; + guint avail, sbpf; + gpointer adata; + gint bpf, rate; + + scope = GST_PM_AUDIO_VISUALIZER(parent); + klass = GST_PM_AUDIO_VISUALIZER_CLASS(G_OBJECT_GET_CLASS(scope)); + + GST_LOG_OBJECT(scope, "chainfunc called"); + + /* resync on DISCONT */ + if (GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DISCONT)) { + gst_adapter_clear(scope->priv->adapter); + } + + /* Make sure have an output format */ + if (gst_pad_check_reconfigure(scope->priv->srcpad)) { + if (!gst_audio_visualizer_src_negotiate(scope)) { + gst_pad_mark_reconfigure(scope->priv->srcpad); + goto not_negotiated; + } + } + + rate = GST_AUDIO_INFO_RATE(&scope->ainfo); + bpf = GST_AUDIO_INFO_BPF(&scope->ainfo); + + if (bpf == 0) { + ret = GST_FLOW_NOT_NEGOTIATED; + goto beach; + } + + gst_adapter_push(scope->priv->adapter, buffer); + + g_mutex_lock(&scope->priv->config_lock); + + /* this is what we want */ + sbpf = scope->req_spf * bpf; + + inbuf = scope->priv->inbuf; + /* FIXME: the timestamp in the adapter would be different */ + gst_buffer_copy_into(inbuf, buffer, GST_BUFFER_COPY_METADATA, 0, -1); + + /* this is what we have */ + avail = gst_adapter_available(scope->priv->adapter); + GST_LOG_OBJECT(scope, "avail: %u, bpf: %u", avail, sbpf); + while (avail >= sbpf) { + GstBuffer *outbuf; + GstVideoFrame outframe; + + /* get timestamp of the current adapter content */ + ts = gst_adapter_prev_pts(scope->priv->adapter, &dist); + if (GST_CLOCK_TIME_IS_VALID(ts)) { + /* convert bytes to time */ + ts += gst_util_uint64_scale_int(dist, GST_SECOND, rate * bpf); + } + + /* check for QoS, don't compute buffers that are known to be late */ + if (GST_CLOCK_TIME_IS_VALID(ts)) { + GstClockTime earliest_time; + gdouble proportion; + gint64 qostime; + + qostime = gst_segment_to_running_time(&scope->priv->segment, + GST_FORMAT_TIME, ts) + + scope->priv->frame_duration; + + GST_OBJECT_LOCK(scope); + earliest_time = scope->priv->earliest_time; + proportion = scope->priv->proportion; + GST_OBJECT_UNLOCK(scope); + + if (GST_CLOCK_TIME_IS_VALID(earliest_time) && qostime <= earliest_time) { + GstClockTime stream_time, jitter; + GstMessage *qos_msg; + + GST_DEBUG_OBJECT(scope, + "QoS: skip ts: %" GST_TIME_FORMAT + ", earliest: %" GST_TIME_FORMAT, + GST_TIME_ARGS(qostime), GST_TIME_ARGS(earliest_time)); + + ++scope->priv->dropped; + stream_time = gst_segment_to_stream_time(&scope->priv->segment, + GST_FORMAT_TIME, ts); + jitter = GST_CLOCK_DIFF(qostime, earliest_time); + qos_msg = + gst_message_new_qos(GST_OBJECT(scope), FALSE, qostime, stream_time, + ts, GST_BUFFER_DURATION(buffer)); + gst_message_set_qos_values(qos_msg, jitter, proportion, 1000000); + gst_message_set_qos_stats(qos_msg, GST_FORMAT_BUFFERS, + scope->priv->processed, scope->priv->dropped); + gst_element_post_message(GST_ELEMENT(scope), qos_msg); + + goto skip; + } + } + + ++scope->priv->processed; + + g_mutex_unlock(&scope->priv->config_lock); + ret = klass->prepare_output_buffer(scope, &outbuf); + g_mutex_lock(&scope->priv->config_lock); + /* recheck as the value could have changed */ + sbpf = scope->req_spf * bpf; + + /* no buffer allocated, we don't care why. */ + if (ret != GST_FLOW_OK) + break; + + // todo ghere + // gst_buffer_add_video_meta(outbuf, GST_VIDEO_FRAME_FLAG_NONE, + // GST_VIDEO_FORMAT_RGB, scope->vinfo.width, scope->vinfo.height); + // gst_video_info_set_format (&scope->vinfo, GST_VIDEO_FORMAT_ARGB, + // scope->vinfo.width, scope->vinfo.height); + /* sync controlled properties */ + if (GST_CLOCK_TIME_IS_VALID(ts)) + gst_object_sync_values(GST_OBJECT(scope), ts); + + GST_BUFFER_PTS(outbuf) = ts; + GST_BUFFER_DURATION(outbuf) = scope->priv->frame_duration; + + /* this can fail as the data size we need could have changed */ + if (!(adata = (gpointer)gst_adapter_map(scope->priv->adapter, sbpf))) + break; + + gst_video_frame_map(&outframe, &scope->vinfo, outbuf, + GST_MAP_WRITE | GST_MAP_GL); + + /* + if (scope->priv->shader) { + gst_video_frame_copy (&outframe, &scope->priv->tempframe); + } else { + // gst_video_frame_clear() or is output frame already cleared + gint i; + + for (i = 0; i < scope->vinfo.finfo->n_planes; i++) { + memset (outframe.data[i], 0, outframe.map[i].size); + } + + } + */ + + gst_buffer_replace_all_memory( + inbuf, gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY, adata, sbpf, 0, + sbpf, NULL, NULL)); + + /* call class->render() vmethod */ + if (klass->render) { + if (!klass->render(scope, inbuf, &outframe)) { + ret = GST_FLOW_ERROR; + gst_video_frame_unmap(&outframe); + goto beach; + } else { + /* run various post processing (shading and geometric transformation) */ + /* FIXME: SHADER assumes 32bpp */ + /* + if (scope->priv->shader && + GST_VIDEO_INFO_COMP_PSTRIDE (&scope->vinfo, 0) == 4) { + scope->priv->shader (scope, &outframe, &scope->priv->tempframe); + }*/ + } + } + gst_video_frame_unmap(&outframe); + + g_mutex_unlock(&scope->priv->config_lock); + ret = gst_pad_push(scope->priv->srcpad, outbuf); + outbuf = NULL; + g_mutex_lock(&scope->priv->config_lock); + + skip: + /* recheck as the value could have changed */ + sbpf = scope->req_spf * bpf; + GST_LOG_OBJECT(scope, "avail: %u, bpf: %u", avail, sbpf); + /* we want to take less or more, depending on spf : req_spf */ + if (avail - sbpf >= sbpf) { + gst_adapter_flush(scope->priv->adapter, sbpf); + gst_adapter_unmap(scope->priv->adapter); + } else if (avail >= sbpf) { + /* just flush a bit and stop */ + gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); + gst_adapter_unmap(scope->priv->adapter); + break; + } + avail = gst_adapter_available(scope->priv->adapter); + + if (ret != GST_FLOW_OK) + break; + } + + g_mutex_unlock(&scope->priv->config_lock); + +beach: + return ret; + + /* ERRORS */ +not_negotiated: { + GST_DEBUG_OBJECT(scope, "Failed to renegotiate"); + return GST_FLOW_NOT_NEGOTIATED; +} +} + +static gboolean gst_audio_visualizer_src_event(GstPad *pad, GstObject *parent, + GstEvent *event) { + gboolean res; + GstPMAudioVisualizer *scope; + + scope = GST_PM_AUDIO_VISUALIZER(parent); + + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_QOS: { + gdouble proportion; + GstClockTimeDiff diff; + GstClockTime timestamp; + + gst_event_parse_qos(event, NULL, &proportion, &diff, ×tamp); + + /* save stuff for the _chain() function */ + GST_OBJECT_LOCK(scope); + scope->priv->proportion = proportion; + if (diff >= 0) + /* we're late, this is a good estimate for next displayable + * frame (see part-qos.txt) */ + scope->priv->earliest_time = + timestamp + 2 * diff + scope->priv->frame_duration; + else + scope->priv->earliest_time = timestamp + diff; + GST_OBJECT_UNLOCK(scope); + + res = gst_pad_push_event(scope->priv->sinkpad, event); + break; + } + case GST_EVENT_RECONFIGURE: + /* don't forward */ + gst_event_unref(event); + res = TRUE; + break; + default: + res = gst_pad_event_default(pad, parent, event); + break; + } + + return res; +} + +static gboolean gst_audio_visualizer_sink_event(GstPad *pad, GstObject *parent, + GstEvent *event) { + gboolean res; + GstPMAudioVisualizer *scope; + + scope = GST_PM_AUDIO_VISUALIZER(parent); + + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_CAPS: { + GstCaps *caps; + + gst_event_parse_caps(event, &caps); + res = gst_audio_visualizer_sink_setcaps(scope, caps); + gst_event_unref(event); + break; + } + case GST_EVENT_FLUSH_STOP: + gst_audio_visualizer_reset(scope); + res = gst_pad_push_event(scope->priv->srcpad, event); + break; + case GST_EVENT_SEGMENT: { + /* the newsegment values are used to clip the input samples + * and to convert the incoming timestamps to running time so + * we can do QoS */ + gst_event_copy_segment(event, &scope->priv->segment); + + res = gst_pad_push_event(scope->priv->srcpad, event); + break; + } + default: + res = gst_pad_event_default(pad, parent, event); + break; + } + + return res; +} + +static gboolean gst_audio_visualizer_src_query(GstPad *pad, GstObject *parent, + GstQuery *query) { + gboolean res = FALSE; + GstPMAudioVisualizer *scope; + + scope = GST_PM_AUDIO_VISUALIZER(parent); + + switch (GST_QUERY_TYPE(query)) { + case GST_QUERY_LATENCY: { + /* We need to send the query upstream and add the returned latency to our + * own */ + GstClockTime min_latency, max_latency; + gboolean us_live; + GstClockTime our_latency; + guint max_samples; + gint rate = GST_AUDIO_INFO_RATE(&scope->ainfo); + + if (rate == 0) + break; + + if ((res = gst_pad_peer_query(scope->priv->sinkpad, query))) { + gst_query_parse_latency(query, &us_live, &min_latency, &max_latency); + + GST_DEBUG_OBJECT( + scope, "Peer latency: min %" GST_TIME_FORMAT " max %" GST_TIME_FORMAT, + GST_TIME_ARGS(min_latency), GST_TIME_ARGS(max_latency)); + + /* the max samples we must buffer buffer */ + max_samples = MAX(scope->req_spf, scope->priv->spf); + our_latency = gst_util_uint64_scale_int(max_samples, GST_SECOND, rate); + + GST_DEBUG_OBJECT(scope, "Our latency: %" GST_TIME_FORMAT, + GST_TIME_ARGS(our_latency)); + + /* we add some latency but only if we need to buffer more than what + * upstream gives us */ + min_latency += our_latency; + if (max_latency != -1) + max_latency += our_latency; + + GST_DEBUG_OBJECT(scope, + "Calculated total latency : min %" GST_TIME_FORMAT + " max %" GST_TIME_FORMAT, + GST_TIME_ARGS(min_latency), GST_TIME_ARGS(max_latency)); + + gst_query_set_latency(query, TRUE, min_latency, max_latency); + } + break; + } + default: + res = gst_pad_query_default(pad, parent, query); + break; + } + + return res; +} + +static GstStateChangeReturn +gst_audio_visualizer_change_state(GstElement *element, + GstStateChange transition) { + GstStateChangeReturn ret; + GstPMAudioVisualizer *scope; + + scope = GST_PM_AUDIO_VISUALIZER(element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + gst_audio_visualizer_reset(scope); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition); + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_audio_visualizer_set_allocation(scope, NULL, NULL, NULL, NULL); + break; + case GST_STATE_CHANGE_READY_TO_NULL: + break; + default: + break; + } + + return ret; +} diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h new file mode 100644 index 0000000..b7e6d64 --- /dev/null +++ b/src/gstpmaudiovisualizer.h @@ -0,0 +1,97 @@ +/* GStreamer + * Copyright (C) <2011> Stefan Kost + * Copyright (C) <2015> Luis de Bethencourt + * + * gstaudiovisualizer.c: base class for audio visualisation elements + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_PM_AUDIO_VISUALIZER_H__ +#define __GST_PM_AUDIO_VISUALIZER_H__ + +#include +#include + +#include +#include +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_PM_AUDIO_VISUALIZER (gst_pm_audio_visualizer_get_type()) +#define GST_PM_AUDIO_VISUALIZER(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_PM_AUDIO_VISUALIZER, \ + GstPMAudioVisualizer)) +#define GST_PM_AUDIO_VISUALIZER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_PM_AUDIO_VISUALIZER, \ + GstPMAudioVisualizerClass)) +#define GST_PM_AUDIO_VISUALIZER_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_PM_AUDIO_VISUALIZER, \ + GstPMAudioVisualizerClass)) +#define GST_IS_SYNAESTHESIA(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_PM_AUDIO_VISUALIZER)) +#define GST_IS_SYNAESTHESIA_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_PM_AUDIO_VISUALIZER)) +typedef struct _GstPMAudioVisualizer GstPMAudioVisualizer; +typedef struct _GstPMAudioVisualizerClass GstPMAudioVisualizerClass; +typedef struct _GstPMAudioVisualizerPrivate GstPMAudioVisualizerPrivate; + +struct _GstPMAudioVisualizer { + GstElement parent; + + guint req_spf; /* min samples per frame wanted by the subclass */ + + /* video state */ + GstVideoInfo vinfo; + + /* audio state */ + GstAudioInfo ainfo; + + /*< private >*/ + GstPMAudioVisualizerPrivate *priv; +}; + +struct _GstPMAudioVisualizerClass { + /*< private >*/ + GstElementClass parent_class; + + /*< public >*/ + /* virtual function, called whenever the format changes */ + gboolean (*setup)(GstPMAudioVisualizer *scope); + + /* virtual function for rendering a frame */ + gboolean (*render)(GstPMAudioVisualizer *scope, GstBuffer *audio, + GstVideoFrame *video); + + gboolean (*decide_allocation)(GstPMAudioVisualizer *scope, GstQuery *query); + + GstFlowReturn (*prepare_output_buffer)(GstPMAudioVisualizer *scope, + GstBuffer **outbuf); +}; + +GST_PBUTILS_API +GType gst_pm_audio_visualizer_get_type(void); + +GST_GL_API +GstFlowReturn +gst_pm_audio_visualizer_prepare_output_buffer(GstPMAudioVisualizer *scope, + GstBuffer **outbuf); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstPMAudioVisualizer, gst_object_unref) + +G_END_DECLS +#endif /* __GST_PM_AUDIO_VISUALIZER_H__ */ diff --git a/src/plugin.c b/src/plugin.c index 125b2ed..178ab1f 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -24,11 +24,16 @@ GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); #define GST_CAT_DEFAULT gst_projectm_debug struct _GstProjectMPrivate { - GLenum gl_format; projectm_handle handle; GstClockTime first_frame_time; gboolean first_frame_received; + + GstGLFramebuffer *fbo; + GLuint textureID; + GstBuffer *in_audio; + GstGLMemory *mem; + GstGLVideoAllocationParams *allocation_params; }; G_DEFINE_TYPE_WITH_CODE(GstProjectM, gst_projectm, @@ -38,6 +43,48 @@ G_DEFINE_TYPE_WITH_CODE(GstProjectM, gst_projectm, "gstprojectm", 0, "Plugin Root")); +static GstBuffer *wrap_gl_texture(GstGLBaseAudioVisualizer *glav, + GstProjectM *plugin) { + GstGLMemoryAllocator *allocator; + gpointer wrapped[1]; + GstGLFormat formats[1]; + GstBuffer *buffer; + gboolean ret; + + allocator = gst_gl_memory_allocator_get_default(glav->context); + + buffer = gst_buffer_new(); + if (!buffer) { + g_error("Failed to create new buffer\n"); + return NULL; + } + + wrapped[0] = (gpointer)plugin->priv->textureID; + formats[0] = GST_GL_RGBA8; + + // * Wrap the texture into GLMemory. * + ret = gst_gl_memory_setup_buffer( + allocator, buffer, plugin->priv->allocation_params, formats, wrapped, 1); + if (!ret) { + g_error("Failed to setup gl memory\n"); + return NULL; + } + + gst_object_unref(allocator); + + return buffer; +} + +static GstFlowReturn +gst_projectm_prepare_output_buffer(GstGLBaseAudioVisualizer *scope, + GstBuffer **outbuf) { + GstProjectM *plugin = GST_PROJECTM(scope); + + *outbuf = wrap_gl_texture(scope, plugin); + GST_INFO_OBJECT(plugin, "Wrapped RT texture buffer"); + return GST_FLOW_OK; +} + void gst_projectm_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { GstProjectM *plugin = GST_PROJECTM(object); @@ -201,6 +248,11 @@ static void gst_projectm_init(GstProjectM *plugin) { plugin->easter_egg = DEFAULT_EASTER_EGG; plugin->preset_locked = DEFAULT_PRESET_LOCKED; plugin->priv->handle = NULL; + plugin->priv->fbo = NULL; + plugin->priv->textureID = 0; + plugin->priv->in_audio = NULL; + plugin->priv->mem = NULL; + plugin->priv->allocation_params = NULL; } static void gst_projectm_finalize(GObject *object) { @@ -217,11 +269,26 @@ static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { projectm_destroy(plugin->priv->handle); plugin->priv->handle = NULL; } + if (plugin->priv->fbo) { + gst_object_unref(plugin->priv->fbo); + plugin->priv->fbo = NULL; + } + + if (plugin->priv->textureID) { + glDeleteTextures(1, &plugin->priv->textureID); + plugin->priv->textureID = 0; + } + + if (plugin->priv->allocation_params) { + gst_gl_allocation_params_free(plugin->priv->allocation_params); + plugin->priv->allocation_params = NULL; + } } static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { // Cast the audio visualizer to the ProjectM plugin GstProjectM *plugin = GST_PROJECTM(glav); + GstPMAudioVisualizer *gstav = GST_PM_AUDIO_VISUALIZER(glav); #ifdef USE_GLEW GST_DEBUG_OBJECT(plugin, "Initializing GLEW"); @@ -232,6 +299,27 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { } #endif + glGenTextures(1, &plugin->priv->textureID); + glBindTexture(GL_TEXTURE_2D, plugin->priv->textureID); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, + GL_UNSIGNED_BYTE, NULL); + // glTexStorage2D (GL_TEXTURE_2D, 1, GL_RGBA8, GST_VIDEO_INFO_WIDTH + // (&gstav->vinfo), GST_VIDEO_INFO_HEIGHT (&gstav->vinfo)); glTexSubImage2D + // (GL_TEXTURE_2D, 0, 0, 0, GST_VIDEO_INFO_WIDTH (&gstav->vinfo), + // GST_VIDEO_INFO_HEIGHT (&gstav->vinfo), GL_RGBA, GL_UNSIGNED_BYTE, NULL); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + // glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE); + glBindTexture(GL_TEXTURE_2D, 0); + + plugin->priv->allocation_params = + gst_gl_video_allocation_params_new_wrapped_texture( + glav->context, NULL, &gstav->vinfo, 0, NULL, GST_GL_TEXTURE_TARGET_2D, + GST_GL_RGBA, plugin->priv->textureID, NULL, 0); + // Check if ProjectM instance exists, and create if not if (!plugin->priv->handle) { // Create ProjectM instance @@ -243,11 +331,55 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { gl_error_handler(glav->context, plugin); } + plugin->priv->fbo = gst_gl_framebuffer_new_with_default_depth( + glav->context, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); + + /* + glBindFramebuffer (GL_FRAMEBUFFER, plugin->priv->fbo->fbo_id); + glBindFramebuffer (GL_FRAMEBUFFER, 0); +*/ + /* + * Color Texture. + * + * IMPORTANT: create a *complete* texture with only one mipmap level. + */ + // glGenTextures (1, &plugin->priv->textureID); + // glBindTexture (GL_TEXTURE_2D, plugin->priv->textureID); + // glTexStorage2D (GL_TEXTURE_2D, 1, GL_RGB8, GST_VIDEO_INFO_WIDTH + // (&gstav->vinfo), GST_VIDEO_INFO_HEIGHT (&gstav->vinfo)); glTexSubImage2D + // (GL_TEXTURE_2D, 0, 0, 0, GST_VIDEO_INFO_WIDTH (&gstav->vinfo), + // GST_VIDEO_INFO_HEIGHT (&gstav->vinfo), GL_RGB, + // GL_UNSIGNED_BYTE, NULL); + // glBindTexture (GL_TEXTURE_2D, 0); + + /* + * Attach empty texture to framebuffer object: drawing to scene_fbo will use + * scene_texture as the backing storage. + */ + // glFramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + // GL_TEXTURE_2D, + // plugin->priv->textureID, 0); + // + // glReadBuffer (GL_COLOR_ATTACHMENT0); + + // GLenum DrawBuffers[1] = { GL_COLOR_ATTACHMENT0 }; + // glDrawBuffers (1, DrawBuffers); + + /* Sanity check. */ + // if (glCheckFramebufferStatus (GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + // { + // g_error("glCheckFramebufferStatus() failed.\n"); + // } + + // glBindFramebuffer (GL_FRAMEBUFFER, 0); + + GST_INFO_OBJECT(plugin, "GL start complete"); return TRUE; } static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { - GstAudioVisualizer *bscope = GST_AUDIO_VISUALIZER(glav); + GstPMAudioVisualizer *bscope = GST_PM_AUDIO_VISUALIZER(glav); GstProjectM *plugin = GST_PROJECTM(glav); // Calculate depth based on pixel stride and bits @@ -262,23 +394,6 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { // format const GstVideoFormat video_format = GST_VIDEO_INFO_FORMAT(&bscope->vinfo); - // TODO: why is the reversed byte order needed when copying pixel data from - // OpenGL ? - switch (video_format) { - case GST_VIDEO_FORMAT_ABGR: - plugin->priv->gl_format = GL_RGBA; - break; - - case GST_VIDEO_FORMAT_RGBA: - // GL_ABGR_EXT does not seem to be well-supported, does not work on Windows - plugin->priv->gl_format = GL_ABGR_EXT; - break; - - default: - GST_ERROR_OBJECT(plugin, "Unsupported video format: %d", video_format); - return FALSE; - } - // Log audio info GST_DEBUG_OBJECT( glav, "Audio Information ", @@ -316,9 +431,9 @@ static double get_seconds_since_first_frame(GstProjectM *plugin, } // TODO: CLEANUP & ADD DEBUGGING -static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, - GstBuffer *audio, GstVideoFrame *video) { - GstProjectM *plugin = GST_PROJECTM(glav); +static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { + GstProjectM *plugin = GST_PROJECTM(stuff); + GstGLBaseAudioVisualizer *gstav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); GstMapInfo audioMap; gboolean result = TRUE; @@ -329,7 +444,7 @@ static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); // AUDIO - gst_buffer_map(audio, &audioMap, GST_MAP_READ); + gst_buffer_map(plugin->priv->in_audio, &audioMap, GST_MAP_READ); // GST_DEBUG_OBJECT(plugin, "Audio Samples: %u, Offset: %lu, Offset End: %lu, // Sample Rate: %d, FPS: %d, Required Samples Per Frame: %d", @@ -344,20 +459,14 @@ static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, // *)audioMap.data)[102], ((gint16 *)audioMap.data)[103]); // VIDEO - const GstGLFuncs *glFunctions = glav->context->gl_vtable; - - size_t windowWidth, windowHeight; - - projectm_get_window_size(plugin->priv->handle, &windowWidth, &windowHeight); + GST_TRACE_OBJECT(plugin, "rendering projectM to fbo %d", + plugin->priv->fbo->fbo_id); + projectm_opengl_render_frame_fbo(plugin->priv->handle, + plugin->priv->fbo->fbo_id); - projectm_opengl_render_frame(plugin->priv->handle); - gl_error_handler(glav->context, plugin); + gl_error_handler(gstav->context, plugin); - glFunctions->ReadPixels(0, 0, windowWidth, windowHeight, - plugin->priv->gl_format, GL_UNSIGNED_INT_8_8_8_8, - (guint8 *)GST_VIDEO_FRAME_PLANE_DATA(video, 0)); - - gst_buffer_unmap(audio, &audioMap); + gst_buffer_unmap(plugin->priv->in_audio, &audioMap); // GST_DEBUG_OBJECT(plugin, "Video Data: %d %d\n", // GST_VIDEO_FRAME_N_PLANES(video), ((uint8_t @@ -368,6 +477,24 @@ static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, return result; } +static gboolean gst_projectm_fill_gl_memory(GstGLBaseAudioVisualizer *glav, + GstBuffer *in_audio, + GstGLMemory *mem) { + + GstProjectM *plugin = GST_PROJECTM(glav); + + plugin->priv->in_audio = in_audio; + plugin->priv->mem = mem; + + gboolean result = gst_gl_framebuffer_draw_to_texture( + plugin->priv->fbo, mem, gst_projectm_fill_gl_memory_callback, plugin); + + plugin->priv->in_audio = NULL; + plugin->priv->mem = NULL; + + return result; +} + static void gst_projectm_class_init(GstProjectMClass *klass) { GObjectClass *gobject_class = (GObjectClass *)klass; GstElementClass *element_class = (GstElementClass *)klass; @@ -497,7 +624,7 @@ static void gst_projectm_class_init(GstProjectMClass *klass) { "preset-locked", "Preset Locked", "Locks or unlocks the current preset. When locked, the visualizer " "remains on the current preset without automatic changes.", - DEFAULT_PRESET_LOCKED, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + DEFAULT_PRESET_LOCKED,G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property( gobject_class, PROP_ENABLE_PLAYLIST, @@ -521,8 +648,10 @@ static void gst_projectm_class_init(GstProjectMClass *klass) { scope_class->supported_gl_api = GST_GL_API_OPENGL3 | GST_GL_API_GLES2; scope_class->gl_start = GST_DEBUG_FUNCPTR(gst_projectm_gl_start); scope_class->gl_stop = GST_DEBUG_FUNCPTR(gst_projectm_gl_stop); - scope_class->gl_render = GST_DEBUG_FUNCPTR(gst_projectm_render); + scope_class->fill_gl_memory = GST_DEBUG_FUNCPTR(gst_projectm_fill_gl_memory); scope_class->setup = GST_DEBUG_FUNCPTR(gst_projectm_setup); + scope_class->prepare_output_buffer = + GST_DEBUG_FUNCPTR(gst_projectm_prepare_output_buffer); } static gboolean plugin_init(GstPlugin *plugin) { diff --git a/src/plugin.h b/src/plugin.h index de1acff..3b7bb5a 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -36,7 +36,7 @@ struct _GstProjectM { }; struct _GstProjectMClass { - GstAudioVisualizerClass parent_class; + GstPMAudioVisualizerClass parent_class; }; static void gst_projectm_set_property(GObject *object, guint prop_id, @@ -53,8 +53,9 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav); static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *glav); -static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, - GstBuffer *audio, GstVideoFrame *video); +static gboolean gst_projectm_fill_gl_memory(GstGLBaseAudioVisualizer *glav, + GstBuffer *in_audio, + GstGLMemory *mem); static void gst_projectm_class_init(GstProjectMClass *klass); diff --git a/src/projectm.c b/src/projectm.c index 1bac137..f6c82c0 100644 --- a/src/projectm.c +++ b/src/projectm.c @@ -15,11 +15,9 @@ GST_DEBUG_CATEGORY_STATIC(projectm_debug); projectm_handle projectm_init(GstProjectM *plugin) { projectm_handle handle = NULL; - projectm_playlist_handle playlist = NULL; + projectm_playlist_handle playlist = NULL;GST_DEBUG_CATEGORY_INIT(projectm_debug, "projectm", 0, "ProjectM"); - GST_DEBUG_CATEGORY_INIT(projectm_debug, "projectm", 0, "ProjectM"); - - GstAudioVisualizer *bscope = GST_AUDIO_VISUALIZER(plugin); + GstPMAudioVisualizer *bscope = GST_PM_AUDIO_VISUALIZER(plugin); // Create ProjectM instance GST_DEBUG_OBJECT(plugin, "Creating projectM instance.."); @@ -44,9 +42,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { // &ProjectMWrapper::PresetSwitchedEvent, static_cast(this)); } else { GST_DEBUG_OBJECT(plugin, "Playlist disabled"); - } - - // Log properties + }// Log properties GST_INFO_OBJECT( plugin, "Using Properties: " @@ -63,8 +59,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { "easter-egg=%f, " "preset-locked=%d, " "enable-playlist=%d, " - "shuffle-presets=%d", - plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, + "shuffle-presets=%d",plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, plugin->hard_cut_duration, plugin->hard_cut_enabled, plugin->hard_cut_sensitivity, plugin->soft_cut_duration, plugin->preset_duration, plugin->mesh_width, plugin->mesh_height, @@ -72,7 +67,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { plugin->enable_playlist, plugin->shuffle_presets); // Load preset file if path is provided - if (plugin->preset_path != NULL) { + if (plugin->preset_path != NULL){ int added_count = projectm_playlist_add_path(playlist, plugin->preset_path, true, false); GST_INFO("Loaded preset path: %s, presets found: %d", plugin->preset_path, @@ -95,8 +90,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { // Set preset duration, or set to in infinite duration if zero if (plugin->preset_duration > 0.0) { projectm_set_preset_duration(handle, plugin->preset_duration); - - // kick off the first preset + // kick off the first preset if (projectm_playlist_size(playlist) > 1 && !plugin->preset_locked) { projectm_playlist_play_next(playlist, true); } From 66e1dcb5b0ac64bb3603393221ca88137ff3467e Mon Sep 17 00:00:00 2001 From: hack Date: Sun, 11 May 2025 01:56:30 -0500 Subject: [PATCH 02/37] integrate pts --- src/gstglbaseaudiovisualizer.c | 26 +++++++++++++++++++++++++- src/gstglbaseaudiovisualizer.h | 3 +++ src/gstpmaudiovisualizer.c | 13 +++++++++++-- src/gstpmaudiovisualizer.h | 9 +++++++++ src/plugin.c | 27 ++------------------------- src/projectm.c | 8 +++++++- 6 files changed, 57 insertions(+), 29 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 097fdd6..b868cbd 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -68,6 +68,8 @@ struct _GstGLBaseAudioVisualizerPrivate { gboolean gl_started; GRecMutex context_lock; + GstClockTime first_frame_time; + gboolean first_frame_received; }; /* Properties */ @@ -170,6 +172,8 @@ static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv->gl_result = TRUE; glav->priv->in_audio = NULL; glav->priv->out_tex = NULL; + glav->priv->first_frame_received = FALSE; + glav->priv->first_frame_time = 0; glav->context = NULL; glav->priv->timestamp_offset = 0; g_rec_mutex_init(&glav->priv->context_lock); @@ -324,6 +328,22 @@ static void _fill_gl(GstGLContext *context, GstGLBaseAudioVisualizer *glav) { klass->fill_gl_memory(glav, glav->priv->in_audio, glav->priv->out_tex); } +static GstClockTime get_time_since_first_frame(GstGLBaseAudioVisualizer *glav, + GstVideoFrame *frame) { + if (!glav->priv->first_frame_received) { + // Store the timestamp of the first frame + glav->priv->first_frame_time = GST_BUFFER_PTS(frame->buffer); + glav->priv->first_frame_received = TRUE; + return 0.0; + } + + // Calculate elapsed time + GstClockTime current_time = GST_BUFFER_PTS(frame->buffer); + GstClockTime elapsed_time = current_time - glav->priv->first_frame_time; + + return elapsed_time; +} + static GstFlowReturn gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, GstGLBaseAudioVisualizer *glav, @@ -361,6 +381,11 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, }*/ glav->priv->out_tex = (GstGLMemory *)video->map[0].memory; + glav->priv->in_audio = audio; + glav->pts = get_time_since_first_frame(glav, video); + + GstBuffer *buffer = video->buffer; + glav->priv->in_audio = audio; gst_gl_context_thread_add(glav->context, (GstGLContextThreadFunc)_fill_gl, @@ -372,7 +397,6 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, if (!glav->priv->gl_result) goto gl_error; - GstBuffer *buffer = video->buffer; sync_meta = gst_buffer_get_gl_sync_meta(buffer); if (sync_meta) diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index 546eb6f..ca1889f 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -81,6 +81,9 @@ struct _GstGLBaseAudioVisualizer { /* total running time */ GstClockTime running_time; + /* current presentatiom time in sec */ + GstClockTime pts; + /*< private >*/ gpointer _padding[GST_PADDING]; diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 1aba979..7b3e38e 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -33,6 +33,14 @@ * new frame. */ +/* + * The code in this file is based on + * GStreamer / gst-plugins-base / 1.19.2: gst-libs/gst/pbutils/gstaudiovisualizer.h + * Git Repository: + * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h + * Original copyright notice has been retained at the top of this file. + */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif @@ -753,10 +761,11 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, if (!(adata = (gpointer)gst_adapter_map(scope->priv->adapter, sbpf))) break; + // projectm patch: modification to allocate GL memory gst_video_frame_map(&outframe, &scope->vinfo, outbuf, GST_MAP_WRITE | GST_MAP_GL); - /* + /* projectm patch: removed cpu based shader if (scope->priv->shader) { gst_video_frame_copy (&outframe, &scope->priv->tempframe); } else { @@ -783,7 +792,7 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, } else { /* run various post processing (shading and geometric transformation) */ /* FIXME: SHADER assumes 32bpp */ - /* + /* projectm patch: removed cpu based shader if (scope->priv->shader && GST_VIDEO_INFO_COMP_PSTRIDE (&scope->vinfo, 0) == 4) { scope->priv->shader (scope, &outframe, &scope->priv->tempframe); diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index b7e6d64..aae7fea 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -20,11 +20,20 @@ * Boston, MA 02110-1301, USA. */ +/* + * The code in this file is based on + * GStreamer / gst-plugins-base / 1.19.2: gst-libs/gst/pbutils/gstaudiovisualizer.h + * Git Repository: + * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h + * Original copyright notice has been retained at the top of this file. + */ + #ifndef __GST_PM_AUDIO_VISUALIZER_H__ #define __GST_PM_AUDIO_VISUALIZER_H__ #include #include +#include #include #include diff --git a/src/plugin.c b/src/plugin.c index 178ab1f..58b0a1f 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -26,9 +26,6 @@ GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); struct _GstProjectMPrivate { projectm_handle handle; - GstClockTime first_frame_time; - gboolean first_frame_received; - GstGLFramebuffer *fbo; GLuint textureID; GstBuffer *in_audio; @@ -411,25 +408,6 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { return TRUE; } -static double get_seconds_since_first_frame(GstProjectM *plugin, - GstVideoFrame *frame) { - if (!plugin->priv->first_frame_received) { - // Store the timestamp of the first frame - plugin->priv->first_frame_time = GST_BUFFER_PTS(frame->buffer); - plugin->priv->first_frame_received = TRUE; - return 0.0; - } - - // Calculate elapsed time - GstClockTime current_time = GST_BUFFER_PTS(frame->buffer); - GstClockTime elapsed_time = current_time - plugin->priv->first_frame_time; - - // Convert to fractional seconds - gdouble elapsed_seconds = (gdouble)elapsed_time / GST_SECOND; - - return elapsed_seconds; -} - // TODO: CLEANUP & ADD DEBUGGING static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstProjectM *plugin = GST_PROJECTM(stuff); @@ -439,9 +417,8 @@ static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { gboolean result = TRUE; // get current gst (PTS) time and set projectM time - double seconds_since_first_frame = - get_seconds_since_first_frame(plugin, video); - projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); + gdouble elapsed_seconds = (gdouble)gstav->pts / GST_SECOND; + projectm_set_frame_time(plugin->priv->handle, elapsed_seconds); // AUDIO gst_buffer_map(plugin->priv->in_audio, &audioMap, GST_MAP_READ); diff --git a/src/projectm.c b/src/projectm.c index f6c82c0..6c1b618 100644 --- a/src/projectm.c +++ b/src/projectm.c @@ -103,7 +103,13 @@ projectm_handle projectm_init(GstProjectM *plugin) { projectm_set_easter_egg(handle, plugin->easter_egg); projectm_set_preset_locked(handle, plugin->preset_locked); - projectm_set_fps(handle, GST_VIDEO_INFO_FPS_N(&bscope->vinfo)); + gdouble fps; + gst_util_fraction_to_double( + GST_VIDEO_INFO_FPS_N(&bscope->vinfo), + GST_VIDEO_INFO_FPS_D(&bscope->vinfo), + &fps); + + projectm_set_fps(handle, gst_util_gdouble_to_guint64(fps)); projectm_set_window_size(handle, GST_VIDEO_INFO_WIDTH(&bscope->vinfo), GST_VIDEO_INFO_HEIGHT(&bscope->vinfo)); From 298d64ae9d9d96add0e4ecacd25d6107508f5065 Mon Sep 17 00:00:00 2001 From: hack Date: Sun, 11 May 2025 01:56:38 -0500 Subject: [PATCH 03/37] update docs --- README.md | 2 +- build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1b43b93..c5afe57 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The documentation has been organized into distinct files, each dedicated to a sp Once the plugin has been installed, you can use it something like this: ```shell -gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false +gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=true ``` Or to convert an audio file to video: diff --git a/build.sh b/build.sh index 74a1148..3fd9d8c 100755 --- a/build.sh +++ b/build.sh @@ -99,7 +99,7 @@ prompt_install() { # Print example command echo echo "Done! Here's an example command:" - echo "gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! $VIDEO_SINK sync=false" + echo "gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! $VIDEO_SINK sync=true" else echo echo "Done!" From b3e81e7600b24e1bdeb13b5440f7ac56ee8ba1cb Mon Sep 17 00:00:00 2001 From: hack Date: Sun, 11 May 2025 01:57:43 -0500 Subject: [PATCH 04/37] format --- src/gstglbaseaudiovisualizer.c | 8 +++----- src/gstpmaudiovisualizer.c | 4 ++-- src/gstpmaudiovisualizer.h | 6 +++--- src/plugin.c | 2 +- src/projectm.c | 18 +++++++++--------- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index b868cbd..6ef3daa 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -158,8 +158,7 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_start); klass->gl_stop = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_stop); - klass->setup = - GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_setup); + klass->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_setup); klass->fill_gl_memory = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_fill_gl_memory); klass->prepare_output_buffer = GST_DEBUG_FUNCPTR( @@ -300,7 +299,7 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, } static GstFlowReturn gst_gl_base_audio_visualizer_default_prepare_output_buffer( - GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf) { + GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf) { GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(scope); return gst_pm_audio_visualizer_prepare_output_buffer(pmav, outbuf); } @@ -329,7 +328,7 @@ static void _fill_gl(GstGLContext *context, GstGLBaseAudioVisualizer *glav) { } static GstClockTime get_time_since_first_frame(GstGLBaseAudioVisualizer *glav, - GstVideoFrame *frame) { + GstVideoFrame *frame) { if (!glav->priv->first_frame_received) { // Store the timestamp of the first frame glav->priv->first_frame_time = GST_BUFFER_PTS(frame->buffer); @@ -397,7 +396,6 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, if (!glav->priv->gl_result) goto gl_error; - sync_meta = gst_buffer_get_gl_sync_meta(buffer); if (sync_meta) gst_gl_sync_meta_set_sync_point(sync_meta, glav->context); diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 7b3e38e..6a8f431 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -35,8 +35,8 @@ /* * The code in this file is based on - * GStreamer / gst-plugins-base / 1.19.2: gst-libs/gst/pbutils/gstaudiovisualizer.h - * Git Repository: + * GStreamer / gst-plugins-base / 1.19.2: + * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h * Original copyright notice has been retained at the top of this file. */ diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index aae7fea..1c2c494 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -22,8 +22,8 @@ /* * The code in this file is based on - * GStreamer / gst-plugins-base / 1.19.2: gst-libs/gst/pbutils/gstaudiovisualizer.h - * Git Repository: + * GStreamer / gst-plugins-base / 1.19.2: + * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h * Original copyright notice has been retained at the top of this file. */ @@ -32,8 +32,8 @@ #define __GST_PM_AUDIO_VISUALIZER_H__ #include -#include #include +#include #include #include diff --git a/src/plugin.c b/src/plugin.c index 58b0a1f..790874b 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -601,7 +601,7 @@ static void gst_projectm_class_init(GstProjectMClass *klass) { "preset-locked", "Preset Locked", "Locks or unlocks the current preset. When locked, the visualizer " "remains on the current preset without automatic changes.", - DEFAULT_PRESET_LOCKED,G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + DEFAULT_PRESET_LOCKED, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property( gobject_class, PROP_ENABLE_PLAYLIST, diff --git a/src/projectm.c b/src/projectm.c index 6c1b618..149a5b6 100644 --- a/src/projectm.c +++ b/src/projectm.c @@ -15,7 +15,8 @@ GST_DEBUG_CATEGORY_STATIC(projectm_debug); projectm_handle projectm_init(GstProjectM *plugin) { projectm_handle handle = NULL; - projectm_playlist_handle playlist = NULL;GST_DEBUG_CATEGORY_INIT(projectm_debug, "projectm", 0, "ProjectM"); + projectm_playlist_handle playlist = NULL; + GST_DEBUG_CATEGORY_INIT(projectm_debug, "projectm", 0, "ProjectM"); GstPMAudioVisualizer *bscope = GST_PM_AUDIO_VISUALIZER(plugin); @@ -42,7 +43,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { // &ProjectMWrapper::PresetSwitchedEvent, static_cast(this)); } else { GST_DEBUG_OBJECT(plugin, "Playlist disabled"); - }// Log properties + } // Log properties GST_INFO_OBJECT( plugin, "Using Properties: " @@ -59,7 +60,8 @@ projectm_handle projectm_init(GstProjectM *plugin) { "easter-egg=%f, " "preset-locked=%d, " "enable-playlist=%d, " - "shuffle-presets=%d",plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, + "shuffle-presets=%d", + plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, plugin->hard_cut_duration, plugin->hard_cut_enabled, plugin->hard_cut_sensitivity, plugin->soft_cut_duration, plugin->preset_duration, plugin->mesh_width, plugin->mesh_height, @@ -67,7 +69,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { plugin->enable_playlist, plugin->shuffle_presets); // Load preset file if path is provided - if (plugin->preset_path != NULL){ + if (plugin->preset_path != NULL) { int added_count = projectm_playlist_add_path(playlist, plugin->preset_path, true, false); GST_INFO("Loaded preset path: %s, presets found: %d", plugin->preset_path, @@ -90,7 +92,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { // Set preset duration, or set to in infinite duration if zero if (plugin->preset_duration > 0.0) { projectm_set_preset_duration(handle, plugin->preset_duration); - // kick off the first preset + // kick off the first preset if (projectm_playlist_size(playlist) > 1 && !plugin->preset_locked) { projectm_playlist_play_next(playlist, true); } @@ -104,10 +106,8 @@ projectm_handle projectm_init(GstProjectM *plugin) { projectm_set_preset_locked(handle, plugin->preset_locked); gdouble fps; - gst_util_fraction_to_double( - GST_VIDEO_INFO_FPS_N(&bscope->vinfo), - GST_VIDEO_INFO_FPS_D(&bscope->vinfo), - &fps); + gst_util_fraction_to_double(GST_VIDEO_INFO_FPS_N(&bscope->vinfo), + GST_VIDEO_INFO_FPS_D(&bscope->vinfo), &fps); projectm_set_fps(handle, gst_util_gdouble_to_guint64(fps)); projectm_set_window_size(handle, GST_VIDEO_INFO_WIDTH(&bscope->vinfo), From 678192d97b27ef8decb95b5ac11c87935f55a381 Mon Sep 17 00:00:00 2001 From: hack Date: Sun, 11 May 2025 02:51:20 -0500 Subject: [PATCH 05/37] cleanup --- src/plugin.c | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 790874b..88801f3 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -332,45 +332,6 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glav->context, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); - /* - glBindFramebuffer (GL_FRAMEBUFFER, plugin->priv->fbo->fbo_id); - glBindFramebuffer (GL_FRAMEBUFFER, 0); -*/ - /* - * Color Texture. - * - * IMPORTANT: create a *complete* texture with only one mipmap level. - */ - // glGenTextures (1, &plugin->priv->textureID); - // glBindTexture (GL_TEXTURE_2D, plugin->priv->textureID); - // glTexStorage2D (GL_TEXTURE_2D, 1, GL_RGB8, GST_VIDEO_INFO_WIDTH - // (&gstav->vinfo), GST_VIDEO_INFO_HEIGHT (&gstav->vinfo)); glTexSubImage2D - // (GL_TEXTURE_2D, 0, 0, 0, GST_VIDEO_INFO_WIDTH (&gstav->vinfo), - // GST_VIDEO_INFO_HEIGHT (&gstav->vinfo), GL_RGB, - // GL_UNSIGNED_BYTE, NULL); - // glBindTexture (GL_TEXTURE_2D, 0); - - /* - * Attach empty texture to framebuffer object: drawing to scene_fbo will use - * scene_texture as the backing storage. - */ - // glFramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, - // GL_TEXTURE_2D, - // plugin->priv->textureID, 0); - // - // glReadBuffer (GL_COLOR_ATTACHMENT0); - - // GLenum DrawBuffers[1] = { GL_COLOR_ATTACHMENT0 }; - // glDrawBuffers (1, DrawBuffers); - - /* Sanity check. */ - // if (glCheckFramebufferStatus (GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) - // { - // g_error("glCheckFramebufferStatus() failed.\n"); - // } - - // glBindFramebuffer (GL_FRAMEBUFFER, 0); - GST_INFO_OBJECT(plugin, "GL start complete"); return TRUE; } From 9a288b933f79de03eddf191c1f3388eb49aba282 Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 20 May 2025 04:17:37 -0500 Subject: [PATCH 06/37] expose pts from audio visualizer plugin clean up --- src/gstglbaseaudiovisualizer.c | 42 ++-------------------------------- src/gstglbaseaudiovisualizer.h | 11 +++++---- src/gstpmaudiovisualizer.c | 12 ++++++---- src/gstpmaudiovisualizer.h | 5 ++++ src/plugin.c | 28 ++++++++++++----------- src/plugin.h | 5 ++++ 6 files changed, 42 insertions(+), 61 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 6ef3daa..481644f 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -243,8 +243,7 @@ static void gst_gl_base_audio_visualizer_set_context(GstElement *element, if (old_display != new_display) { gst_clear_object(&glav->context); if (gst_gl_base_audio_visualizer_find_gl_context_unlocked(glav)) { - // TODO does this need to be handled ? - // gst_pad_mark_reconfigure (GST_BASE_SRC_PAD (glav)); + gst_pad_mark_reconfigure(GST_BASE_SRC_PAD(glav)); } } } @@ -270,7 +269,6 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(data); GstGLBaseAudioVisualizerClass *glav_class = GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(glav); - GstPMAudioVisualizer *gstav = GST_PM_AUDIO_VISUALIZER(glav); GST_INFO_OBJECT(glav, "starting"); gst_gl_insert_debug_marker(glav->context, "starting element %s", @@ -327,22 +325,6 @@ static void _fill_gl(GstGLContext *context, GstGLBaseAudioVisualizer *glav) { klass->fill_gl_memory(glav, glav->priv->in_audio, glav->priv->out_tex); } -static GstClockTime get_time_since_first_frame(GstGLBaseAudioVisualizer *glav, - GstVideoFrame *frame) { - if (!glav->priv->first_frame_received) { - // Store the timestamp of the first frame - glav->priv->first_frame_time = GST_BUFFER_PTS(frame->buffer); - glav->priv->first_frame_received = TRUE; - return 0.0; - } - - // Calculate elapsed time - GstClockTime current_time = GST_BUFFER_PTS(frame->buffer); - GstClockTime elapsed_time = current_time - glav->priv->first_frame_time; - - return elapsed_time; -} - static GstFlowReturn gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, GstGLBaseAudioVisualizer *glav, @@ -359,29 +341,9 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, glav->priv->n_frames == 1)) goto eos; - // there is an issue here: the video buffer has already been mapped without - // GST_MAP_GL flag - - /* - GstBuffer *buffer; - - // Allocate a buffer of specific size - buffer = gst_gl_buffer_new_wrapped (g_malloc0 (bscope->vinfo.size), - bscope->vinfo.size); - - // Map the buffer to set its memory data - GstVideoFrame map; - - //gst_video_frame_unmap (video); - - if (!gst_video_frame_map (&map, &bscope->vinfo, buffer, - GST_MAP_WRITE | GST_MAP_GL)) { - return GST_FLOW_NOT_NEGOTIATED; - }*/ - + // video is mapped to gl memory glav->priv->out_tex = (GstGLMemory *)video->map[0].memory; glav->priv->in_audio = audio; - glav->pts = get_time_since_first_frame(glav, video); GstBuffer *buffer = video->buffer; diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index ca1889f..c118eb6 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -78,12 +78,10 @@ struct _GstGLBaseAudioVisualizer { GstGLDisplay *display; GstGLContext *context; - /* total running time */ + /* buffer running time (determined by no. of frames rendered). clock for + * buffer position. */ GstClockTime running_time; - /* current presentatiom time in sec */ - GstClockTime pts; - /*< private >*/ gpointer _padding[GST_PADDING]; @@ -107,11 +105,16 @@ struct _GstGLBaseAudioVisualizerClass { /*< public >*/ GstGLAPI supported_gl_api; + /* called when gl context starts */ gboolean (*gl_start)(GstGLBaseAudioVisualizer *glav); + /* called when gl context stops */ void (*gl_stop)(GstGLBaseAudioVisualizer *glav); + /* called once for the pipeline in the beginning */ gboolean (*setup)(GstGLBaseAudioVisualizer *glav); + /* called to render each frame */ gboolean (*fill_gl_memory)(GstGLBaseAudioVisualizer *glav, GstBuffer *in_audio, GstGLMemory *mem); + /* allocate buffer for frame rendering */ GstFlowReturn (*prepare_output_buffer)(GstGLBaseAudioVisualizer *glav, GstBuffer **outbuf); diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 6a8f431..0b794b0 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -39,6 +39,9 @@ * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h * Original copyright notice has been retained at the top of this file. + * + * The code has been modified to map gl memory for the video output buffer and + * expose pts running_time. Support for CPU based shaders has been removed. */ #ifdef HAVE_CONFIG_H @@ -697,11 +700,12 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, if (GST_CLOCK_TIME_IS_VALID(ts)) { GstClockTime earliest_time; gdouble proportion; - gint64 qostime; + guint64 qostime; + + scope->running_time = gst_segment_to_running_time(&scope->priv->segment, + GST_FORMAT_TIME, ts); - qostime = gst_segment_to_running_time(&scope->priv->segment, - GST_FORMAT_TIME, ts) + - scope->priv->frame_duration; + qostime = scope->running_time + scope->priv->frame_duration; GST_OBJECT_LOCK(scope); earliest_time = scope->priv->earliest_time; diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index 1c2c494..a892069 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -70,6 +70,9 @@ struct _GstPMAudioVisualizer { /* audio state */ GstAudioInfo ainfo; + /* current pts running time for syncing renderers */ + guint64 running_time; + /*< private >*/ GstPMAudioVisualizerPrivate *priv; }; @@ -86,8 +89,10 @@ struct _GstPMAudioVisualizerClass { gboolean (*render)(GstPMAudioVisualizer *scope, GstBuffer *audio, GstVideoFrame *video); + /* virtual function for gl buffer pool allocation */ gboolean (*decide_allocation)(GstPMAudioVisualizer *scope, GstQuery *query); + /* virtual function for output buffer allocation */ GstFlowReturn (*prepare_output_buffer)(GstPMAudioVisualizer *scope, GstBuffer **outbuf); }; diff --git a/src/plugin.c b/src/plugin.c index 88801f3..7744176 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -298,13 +298,18 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glGenTextures(1, &plugin->priv->textureID); glBindTexture(GL_TEXTURE_2D, plugin->priv->textureID); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, - GL_UNSIGNED_BYTE, NULL); - // glTexStorage2D (GL_TEXTURE_2D, 1, GL_RGBA8, GST_VIDEO_INFO_WIDTH - // (&gstav->vinfo), GST_VIDEO_INFO_HEIGHT (&gstav->vinfo)); glTexSubImage2D - // (GL_TEXTURE_2D, 0, 0, 0, GST_VIDEO_INFO_WIDTH (&gstav->vinfo), - // GST_VIDEO_INFO_HEIGHT (&gstav->vinfo), GL_RGBA, GL_UNSIGNED_BYTE, NULL); + + /* glTexImage2D cloud be used if needed */ + // glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, + // GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + // GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, + // GL_UNSIGNED_BYTE, NULL); + + // use immutable texture buffer + glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, + GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); @@ -348,10 +353,6 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { bscope->req_spf = (bscope->ainfo.channels * bscope->ainfo.rate * 2) / bscope->vinfo.fps_n; - // get GStreamer video format and map it to the corresponding OpenGL pixel - // format - const GstVideoFormat video_format = GST_VIDEO_INFO_FORMAT(&bscope->vinfo); - // Log audio info GST_DEBUG_OBJECT( glav, "Audio Information ", @@ -373,13 +374,14 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstProjectM *plugin = GST_PROJECTM(stuff); GstGLBaseAudioVisualizer *gstav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(stuff); GstMapInfo audioMap; gboolean result = TRUE; // get current gst (PTS) time and set projectM time - gdouble elapsed_seconds = (gdouble)gstav->pts / GST_SECOND; - projectm_set_frame_time(plugin->priv->handle, elapsed_seconds); + gdouble seconds_since_first_frame = (gdouble)pmav->running_time / GST_SECOND; + projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); // AUDIO gst_buffer_map(plugin->priv->in_audio, &audioMap, GST_MAP_READ); diff --git a/src/plugin.h b/src/plugin.h index 3b7bb5a..4210fd0 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -12,6 +12,11 @@ G_BEGIN_DECLS G_DECLARE_FINAL_TYPE(GstProjectM, gst_projectm, GST, PROJECTM, GstGLBaseAudioVisualizer) +/* + * Main plug-in. Handles interactions with projectM. Extends plug-ins for gl + * context handling and audio-visualization (timing, video frame data). + * GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. + */ struct _GstProjectM { GstGLBaseAudioVisualizer element; From 1669200093302ec62d4feb0f31773586255950bb Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 20 May 2025 04:22:08 -0500 Subject: [PATCH 07/37] fix warnings --- src/plugin.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 7744176..3671092 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -8,7 +8,6 @@ #endif #include #include -#include #include @@ -277,7 +276,7 @@ static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { } if (plugin->priv->allocation_params) { - gst_gl_allocation_params_free(plugin->priv->allocation_params); + gst_gl_video_allocation_params_free_data(plugin->priv->allocation_params); plugin->priv->allocation_params = NULL; } } From a39467270babebc701d21dc31fbe969bf967b971 Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 20 May 2025 12:24:58 -0500 Subject: [PATCH 08/37] keep using glTexImage2D --- src/plugin.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 3671092..c39440b 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -298,16 +298,16 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glGenTextures(1, &plugin->priv->textureID); glBindTexture(GL_TEXTURE_2D, plugin->priv->textureID); - /* glTexImage2D cloud be used if needed */ - // glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, - // GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - // GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, - // GL_UNSIGNED_BYTE, NULL); - - // use immutable texture buffer - glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, - GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); + /* allocate texture using glTexImage2D */ + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, + GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, + GL_UNSIGNED_BYTE, NULL); + + /* could use immutable texture buffer, but it's GL4 / ES3 and not supported on mac os x */ + //glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, + // GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + // GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); From 81ff58fab5bcbe6edf82535d15702424a43a533c Mon Sep 17 00:00:00 2001 From: hack Date: Wed, 21 May 2025 23:59:48 -0500 Subject: [PATCH 09/37] cleanup --- src/gstglbaseaudiovisualizer.c | 33 ++-- src/gstglbaseaudiovisualizer.h | 5 - src/gstpmaudiovisualizer.c | 331 ++++++++++++++------------------- src/gstpmaudiovisualizer.h | 19 +- src/plugin.c | 18 +- 5 files changed, 176 insertions(+), 230 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 481644f..2b4f53e 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -62,14 +62,15 @@ struct _GstGLBaseAudioVisualizerPrivate { GstGLMemory *out_tex; GstBuffer *in_audio; - gint64 timestamp_offset; /* base offset */ - gint64 n_frames; /* total frames sent */ + gint64 timestamp_offset; /* base offset */ + gint64 n_frames; /* total frames sent */ + GstClockTime buf_running_time; /* determined by no. of frames rendered. clock + for buffer position. */ + gboolean gl_result; gboolean gl_started; GRecMutex context_lock; - GstClockTime first_frame_time; - gboolean first_frame_received; }; /* Properties */ @@ -129,6 +130,9 @@ static GstFlowReturn gst_gl_base_audio_visualizer_default_prepare_output_buffer( static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( GstPMAudioVisualizer *scope, GstBuffer **outbuf); +static void gst_gl_base_audio_visualizer_map_output_buffer( + GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf); + static void gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); @@ -152,6 +156,8 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { gstav_class->render = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_render); gstav_class->prepare_output_buffer = GST_DEBUG_FUNCPTR( gst_gl_base_audio_visualizer_parent_prepare_output_buffer); + gstav_class->map_output_buffer = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_map_output_buffer); klass->supported_gl_api = GST_GL_API_ANY; klass->gl_start = @@ -171,8 +177,6 @@ static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv->gl_result = TRUE; glav->priv->in_audio = NULL; glav->priv->out_tex = NULL; - glav->priv->first_frame_received = FALSE; - glav->priv->first_frame_time = 0; glav->context = NULL; glav->priv->timestamp_offset = 0; g_rec_mutex_init(&glav->priv->context_lock); @@ -299,7 +303,7 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, static GstFlowReturn gst_gl_base_audio_visualizer_default_prepare_output_buffer( GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf) { GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(scope); - return gst_pm_audio_visualizer_prepare_output_buffer(pmav, outbuf); + return gst_pm_audio_visualizer_default_prepare_output_buffer(pmav, outbuf); } static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( @@ -310,6 +314,13 @@ static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( return klass->prepare_output_buffer(glav, outbuf); } +static void gst_gl_base_audio_visualizer_map_output_buffer( + GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf) { + /* map video to gl memory */ + gst_video_frame_map(outframe, &scope->vinfo, outbuf, + GST_MAP_WRITE | GST_MAP_GL); +} + static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( GstGLBaseAudioVisualizer *glav, GstBuffer *in_audio, GstGLMemory *mem) { return TRUE; @@ -365,7 +376,7 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, g_rec_mutex_unlock(&glav->priv->context_lock); GST_BUFFER_TIMESTAMP(buffer) = - glav->priv->timestamp_offset + glav->running_time; + glav->priv->timestamp_offset + glav->priv->buf_running_time; GST_BUFFER_OFFSET(buffer) = glav->priv->n_frames; glav->priv->n_frames++; GST_BUFFER_OFFSET_END(buffer) = glav->priv->n_frames; @@ -373,14 +384,14 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, next_time = gst_util_uint64_scale_int(glav->priv->n_frames * GST_SECOND, bscope->vinfo.fps_d, bscope->vinfo.fps_n); - GST_BUFFER_DURATION(buffer) = next_time - glav->running_time; + GST_BUFFER_DURATION(buffer) = next_time - glav->priv->buf_running_time; } else { next_time = glav->priv->timestamp_offset; /* NONE means forever */ GST_BUFFER_DURATION(buffer) = GST_CLOCK_TIME_NONE; } - glav->running_time = next_time; + glav->priv->buf_running_time = next_time; return GST_FLOW_OK; @@ -427,7 +438,7 @@ gst_gl_base_audio_visualizer_render(GstPMAudioVisualizer *bscope, static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav) { glav->priv->n_frames = 0; - glav->running_time = 0; + glav->priv->buf_running_time = 0; } static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav) { diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index c118eb6..dacdc5d 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -34,7 +34,6 @@ #include "gstpmaudiovisualizer.h" #include -#include #include typedef struct _GstGLBaseAudioVisualizer GstGLBaseAudioVisualizer; @@ -78,10 +77,6 @@ struct _GstGLBaseAudioVisualizer { GstGLDisplay *display; GstGLContext *context; - /* buffer running time (determined by no. of frames rendered). clock for - * buffer position. */ - GstClockTime running_time; - /*< private >*/ gpointer _padding[GST_PADDING]; diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 0b794b0..851851e 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -59,8 +59,8 @@ #include #include -GST_DEBUG_CATEGORY_STATIC(audio_visualizer_debug); -#define GST_CAT_DEFAULT (audio_visualizer_debug) +GST_DEBUG_CATEGORY_STATIC(pm_audio_visualizer_debug); +#define GST_CAT_DEFAULT (pm_audio_visualizer_debug) #define DEFAULT_SHADER GST_AUDIO_VISUALIZER_SHADER_FADE #define DEFAULT_SHADE_AMOUNT 0x000a0a0a @@ -70,42 +70,55 @@ enum { PROP_0 }; static GstBaseTransformClass *parent_class = NULL; static gint private_offset = 0; -static void gst_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass); -static void gst_audio_visualizer_init(GstPMAudioVisualizer *scope, - GstPMAudioVisualizerClass *g_class); -static void gst_audio_visualizer_set_property(GObject *object, guint prop_id, - const GValue *value, - GParamSpec *pspec); -static void gst_audio_visualizer_get_property(GObject *object, guint prop_id, - GValue *value, GParamSpec *pspec); -static void gst_audio_visualizer_dispose(GObject *object); - -static gboolean gst_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope); -static gboolean gst_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, - GstCaps *caps); -static gboolean gst_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, - GstCaps *caps); - -static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, - GstBuffer *buffer); - -static gboolean gst_audio_visualizer_src_event(GstPad *pad, GstObject *parent, - GstEvent *event); -static gboolean gst_audio_visualizer_sink_event(GstPad *pad, GstObject *parent, - GstEvent *event); - -static gboolean gst_audio_visualizer_src_query(GstPad *pad, GstObject *parent, - GstQuery *query); +static void +gst_pm_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass); +static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, + GstPMAudioVisualizerClass *g_class); +static void gst_pm_audio_visualizer_set_property(GObject *object, guint prop_id, + const GValue *value, + GParamSpec *pspec); +static void gst_pm_audio_visualizer_get_property(GObject *object, guint prop_id, + GValue *value, + GParamSpec *pspec); +static void gst_pm_audio_visualizer_dispose(GObject *object); + +static gboolean +gst_pm_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope); +static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps); +static gboolean +gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps); + +static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, + GstObject *parent, + GstBuffer *buffer); + +static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, + GstObject *parent, + GstEvent *event); +static gboolean gst_pm_audio_visualizer_sink_event(GstPad *pad, + GstObject *parent, + GstEvent *event); + +static gboolean gst_pm_audio_visualizer_src_query(GstPad *pad, + GstObject *parent, + GstQuery *query); static GstStateChangeReturn -gst_audio_visualizer_change_state(GstElement *element, - GstStateChange transition); +gst_pm_audio_visualizer_change_state(GstElement *element, + GstStateChange transition); -static gboolean gst_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, - GstCaps *outcaps); +static gboolean +gst_pm_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, + GstCaps *outcaps); -static gboolean default_decide_allocation(GstPMAudioVisualizer *scope, - GstQuery *query); +static gboolean +gst_pm_audio_visualizer_default_decide_allocation(GstPMAudioVisualizer *scope, + GstQuery *query); + +static void gst_pm_audio_visualizer_default_map_output_buffer( + GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf); struct _GstPMAudioVisualizerPrivate { gboolean negotiated; @@ -122,8 +135,6 @@ struct _GstPMAudioVisualizerPrivate { GstAdapter *adapter; GstBuffer *inbuf; - GstBuffer *tempbuf; - GstVideoFrame tempframe; guint spf; /* samples per video frame */ guint64 frame_duration; @@ -151,12 +162,12 @@ GType gst_pm_audio_visualizer_get_type(void) { sizeof(GstPMAudioVisualizerClass), NULL, NULL, - (GClassInitFunc)gst_audio_visualizer_class_init, + (GClassInitFunc)gst_pm_audio_visualizer_class_init, NULL, NULL, sizeof(GstPMAudioVisualizer), 0, - (GInstanceInitFunc)gst_audio_visualizer_init, + (GInstanceInitFunc)gst_pm_audio_visualizer_init, }; GType _type; @@ -178,7 +189,8 @@ gst_audio_visualizer_get_instance_private(GstPMAudioVisualizer *self) { return (G_STRUCT_MEMBER_P(self, private_offset)); } -static void gst_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass) { +static void +gst_pm_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass) { GObjectClass *gobject_class = (GObjectClass *)klass; GstElementClass *element_class = (GstElementClass *)klass; @@ -187,24 +199,26 @@ static void gst_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass) { parent_class = g_type_class_peek_parent(klass); - GST_DEBUG_CATEGORY_INIT(audio_visualizer_debug, - "baseaudiovisualizer-libvisual", 0, - "scope audio visualisation base class"); + GST_DEBUG_CATEGORY_INIT(pm_audio_visualizer_debug, "pmaudiovisualizer", 0, + "projectm audio visualisation base class"); - gobject_class->set_property = gst_audio_visualizer_set_property; - gobject_class->get_property = gst_audio_visualizer_get_property; - gobject_class->dispose = gst_audio_visualizer_dispose; + gobject_class->set_property = gst_pm_audio_visualizer_set_property; + gobject_class->get_property = gst_pm_audio_visualizer_get_property; + gobject_class->dispose = gst_pm_audio_visualizer_dispose; element_class->change_state = - GST_DEBUG_FUNCPTR(gst_audio_visualizer_change_state); + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_change_state); - klass->decide_allocation = GST_DEBUG_FUNCPTR(default_decide_allocation); + klass->decide_allocation = + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_default_decide_allocation); klass->prepare_output_buffer = - GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_prepare_output_buffer); + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_default_prepare_output_buffer); + klass->map_output_buffer = + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_default_map_output_buffer); } -static void gst_audio_visualizer_init(GstPMAudioVisualizer *scope, - GstPMAudioVisualizerClass *g_class) { +static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, + GstPMAudioVisualizerClass *g_class) { GstPadTemplate *pad_template; scope->priv = gst_audio_visualizer_get_instance_private(scope); @@ -215,19 +229,22 @@ static void gst_audio_visualizer_init(GstPMAudioVisualizer *scope, g_return_if_fail(pad_template != NULL); scope->priv->sinkpad = gst_pad_new_from_template(pad_template, "sink"); gst_pad_set_chain_function(scope->priv->sinkpad, - GST_DEBUG_FUNCPTR(gst_audio_visualizer_chain)); + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_chain)); gst_pad_set_event_function( - scope->priv->sinkpad, GST_DEBUG_FUNCPTR(gst_audio_visualizer_sink_event)); + scope->priv->sinkpad, + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_sink_event)); gst_element_add_pad(GST_ELEMENT(scope), scope->priv->sinkpad); pad_template = gst_element_class_get_pad_template(GST_ELEMENT_CLASS(g_class), "src"); g_return_if_fail(pad_template != NULL); scope->priv->srcpad = gst_pad_new_from_template(pad_template, "src"); - gst_pad_set_event_function(scope->priv->srcpad, - GST_DEBUG_FUNCPTR(gst_audio_visualizer_src_event)); - gst_pad_set_query_function(scope->priv->srcpad, - GST_DEBUG_FUNCPTR(gst_audio_visualizer_src_query)); + gst_pad_set_event_function( + scope->priv->srcpad, + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_src_event)); + gst_pad_set_query_function( + scope->priv->srcpad, + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_src_query)); gst_element_add_pad(GST_ELEMENT(scope), scope->priv->srcpad); scope->priv->adapter = gst_adapter_new(); @@ -246,9 +263,9 @@ static void gst_audio_visualizer_init(GstPMAudioVisualizer *scope, g_mutex_init(&scope->priv->config_lock); } -static void gst_audio_visualizer_set_property(GObject *object, guint prop_id, - const GValue *value, - GParamSpec *pspec) { +static void gst_pm_audio_visualizer_set_property(GObject *object, guint prop_id, + const GValue *value, + GParamSpec *pspec) { GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); switch (prop_id) { @@ -258,9 +275,9 @@ static void gst_audio_visualizer_set_property(GObject *object, guint prop_id, } } -static void gst_audio_visualizer_get_property(GObject *object, guint prop_id, - GValue *value, - GParamSpec *pspec) { +static void gst_pm_audio_visualizer_get_property(GObject *object, guint prop_id, + GValue *value, + GParamSpec *pspec) { GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); switch (prop_id) { @@ -270,7 +287,7 @@ static void gst_audio_visualizer_get_property(GObject *object, guint prop_id, } } -static void gst_audio_visualizer_dispose(GObject *object) { +static void gst_pm_audio_visualizer_dispose(GObject *object) { GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); if (scope->priv->adapter) { @@ -281,11 +298,6 @@ static void gst_audio_visualizer_dispose(GObject *object) { gst_buffer_unref(scope->priv->inbuf); scope->priv->inbuf = NULL; } - if (scope->priv->tempbuf) { - gst_video_frame_unmap(&scope->priv->tempframe); - gst_buffer_unref(scope->priv->tempbuf); - scope->priv->tempbuf = NULL; - } if (scope->priv->config_lock.p) { g_mutex_clear(&scope->priv->config_lock); scope->priv->config_lock.p = NULL; @@ -305,8 +317,9 @@ static void gst_audio_visualizer_reset(GstPMAudioVisualizer *scope) { GST_OBJECT_UNLOCK(scope); } -static gboolean gst_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, - GstCaps *caps) { +static gboolean +gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps) { GstAudioInfo info; if (!gst_audio_info_from_caps(&info, caps)) @@ -317,7 +330,7 @@ static gboolean gst_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, GST_DEBUG_OBJECT(scope, "audio: channels %d, rate %d", GST_AUDIO_INFO_CHANNELS(&info), GST_AUDIO_INFO_RATE(&info)); - if (!gst_audio_visualizer_src_negotiate(scope)) { + if (!gst_pm_audio_visualizer_src_negotiate(scope)) { goto not_negotiated; } @@ -334,8 +347,8 @@ not_negotiated: { } } -static gboolean gst_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, - GstCaps *caps) { +static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps) { GstVideoInfo info; GstPMAudioVisualizerClass *klass; gboolean res; @@ -354,15 +367,6 @@ static gboolean gst_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, GST_VIDEO_INFO_FPS_N(&info)); scope->req_spf = scope->priv->spf; - if (scope->priv->tempbuf) { - gst_video_frame_unmap(&scope->priv->tempframe); - gst_buffer_unref(scope->priv->tempbuf); - } - scope->priv->tempbuf = - gst_buffer_new_wrapped(g_malloc0(scope->vinfo.size), scope->vinfo.size); - gst_video_frame_map(&scope->priv->tempframe, &scope->vinfo, - scope->priv->tempbuf, GST_MAP_READWRITE); - if (klass->setup && !klass->setup(scope)) goto setup_failed; @@ -375,7 +379,7 @@ static gboolean gst_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, gst_pad_set_caps(scope->priv->srcpad, caps); /* find a pool for the negotiated caps now */ - res = gst_audio_visualizer_do_bufferpool(scope, caps); + res = gst_pm_audio_visualizer_do_bufferpool(scope, caps); gst_caps_unref(caps); return res; @@ -394,7 +398,7 @@ setup_failed: { } static gboolean -gst_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope) { +gst_pm_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope) { GstCaps *othercaps, *target; GstStructure *structure; GstCaps *templ; @@ -432,7 +436,7 @@ gst_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope) { GST_DEBUG_OBJECT(scope, "final caps are %" GST_PTR_FORMAT, target); - ret = gst_audio_visualizer_src_setcaps(scope, target); + ret = gst_pm_audio_visualizer_src_setcaps(scope, target); return ret; @@ -482,8 +486,9 @@ static gboolean gst_audio_visualizer_set_allocation( return TRUE; } -static gboolean gst_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, - GstCaps *outcaps) { +static gboolean +gst_pm_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, + GstCaps *outcaps) { GstQuery *query; gboolean result = TRUE; GstBufferPool *pool = NULL; @@ -540,74 +545,25 @@ no_decide_allocation: { } } -static gboolean default_decide_allocation(GstPMAudioVisualizer *scope, - GstQuery *query) { - GstCaps *outcaps; - GstBufferPool *pool; - guint size, min, max; - GstAllocator *allocator; - GstAllocationParams params; - GstStructure *config; - gboolean update_allocator; - gboolean update_pool; - - gst_query_parse_allocation(query, &outcaps, NULL); - - /* we got configuration from our peer or the decide_allocation method, - * parse them */ - if (gst_query_get_n_allocation_params(query) > 0) { - /* try the allocator */ - gst_query_parse_nth_allocation_param(query, 0, &allocator, ¶ms); - update_allocator = TRUE; - } else { - allocator = NULL; - gst_allocation_params_init(¶ms); - update_allocator = FALSE; - } - - if (gst_query_get_n_allocation_pools(query) > 0) { - gst_query_parse_nth_allocation_pool(query, 0, &pool, &size, &min, &max); - update_pool = TRUE; - } else { - pool = NULL; - size = GST_VIDEO_INFO_SIZE(&scope->vinfo); - min = max = 0; - update_pool = FALSE; - } - - if (pool == NULL) { - /* we did not get a pool, make one ourselves then */ - pool = gst_video_buffer_pool_new(); - } - - config = gst_buffer_pool_get_config(pool); - gst_buffer_pool_config_set_params(config, outcaps, size, min, max); - gst_buffer_pool_config_set_allocator(config, allocator, ¶ms); - gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_META); - gst_buffer_pool_set_config(pool, config); - - if (update_allocator) - gst_query_set_nth_allocation_param(query, 0, allocator, ¶ms); - else - gst_query_add_allocation_param(query, allocator, ¶ms); - - if (allocator) - gst_object_unref(allocator); - - if (update_pool) - gst_query_set_nth_allocation_pool(query, 0, pool, size, min, max); - else - gst_query_add_allocation_pool(query, pool, size, min, max); - - if (pool) - gst_object_unref(pool); +static gboolean +gst_pm_audio_visualizer_default_decide_allocation(GstPMAudioVisualizer *scope, + GstQuery *query) { + /* removed main memory pool implementation. This vmethod is overridden for + * using gl memory by gstglbaseaudiovisualizer. */ + g_error("vmethod gst_pm_audio_visualizer_default_decide_allocation is not " + "implemented"); +} - return TRUE; +static void gst_pm_audio_visualizer_default_map_output_buffer( + GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf) { + /* removed main memory buffer implementation. This vmethod is overridden for + * using gl memory by gstglbaseaudiovisualizer. */ + g_error("vmethod gst_pm_audio_visualizer_default_map_output_buffer is not " + "implemented"); } -GstFlowReturn -gst_pm_audio_visualizer_prepare_output_buffer(GstPMAudioVisualizer *scope, - GstBuffer **outbuf) { +GstFlowReturn gst_pm_audio_visualizer_default_prepare_output_buffer( + GstPMAudioVisualizer *scope, GstBuffer **outbuf) { GstPMAudioVisualizerPrivate *priv; priv = scope->priv; @@ -634,8 +590,9 @@ activate_failed: { } } -static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, - GstBuffer *buffer) { +static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, + GstObject *parent, + GstBuffer *buffer) { GstFlowReturn ret = GST_FLOW_OK; GstPMAudioVisualizer *scope; GstPMAudioVisualizerClass *klass; @@ -657,7 +614,7 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, /* Make sure have an output format */ if (gst_pad_check_reconfigure(scope->priv->srcpad)) { - if (!gst_audio_visualizer_src_negotiate(scope)) { + if (!gst_pm_audio_visualizer_src_negotiate(scope)) { gst_pad_mark_reconfigure(scope->priv->srcpad); goto not_negotiated; } @@ -689,7 +646,7 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, GstBuffer *outbuf; GstVideoFrame outframe; - /* get timestamp of the current adapter content */ + /* get timestamp of the current adapter content (audio input timestamp) */ ts = gst_adapter_prev_pts(scope->priv->adapter, &dist); if (GST_CLOCK_TIME_IS_VALID(ts)) { /* convert bytes to time */ @@ -749,11 +706,15 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, if (ret != GST_FLOW_OK) break; - // todo ghere - // gst_buffer_add_video_meta(outbuf, GST_VIDEO_FRAME_FLAG_NONE, - // GST_VIDEO_FORMAT_RGB, scope->vinfo.width, scope->vinfo.height); - // gst_video_info_set_format (&scope->vinfo, GST_VIDEO_FORMAT_ARGB, - // scope->vinfo.width, scope->vinfo.height); + /* + todo: need to set any buffer meta for gl ? + gst_buffer_add_video_meta(outbuf, GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_FORMAT_RGBA, scope->vinfo.width, + scope->vinfo.height); + gst_video_info_set_format (&scope->vinfo, GST_VIDEO_FORMAT_RGBA, + scope->vinfo.width, scope->vinfo.height); + */ + /* sync controlled properties */ if (GST_CLOCK_TIME_IS_VALID(ts)) gst_object_sync_values(GST_OBJECT(scope), ts); @@ -765,23 +726,8 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, if (!(adata = (gpointer)gst_adapter_map(scope->priv->adapter, sbpf))) break; - // projectm patch: modification to allocate GL memory - gst_video_frame_map(&outframe, &scope->vinfo, outbuf, - GST_MAP_WRITE | GST_MAP_GL); - - /* projectm patch: removed cpu based shader - if (scope->priv->shader) { - gst_video_frame_copy (&outframe, &scope->priv->tempframe); - } else { - // gst_video_frame_clear() or is output frame already cleared - gint i; - - for (i = 0; i < scope->vinfo.finfo->n_planes; i++) { - memset (outframe.data[i], 0, outframe.map[i].size); - } - - } - */ + // projectm patch: modification to customize mapping + klass->map_output_buffer(scope, &outframe, outbuf); gst_buffer_replace_all_memory( inbuf, gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY, adata, sbpf, 0, @@ -793,14 +739,6 @@ static GstFlowReturn gst_audio_visualizer_chain(GstPad *pad, GstObject *parent, ret = GST_FLOW_ERROR; gst_video_frame_unmap(&outframe); goto beach; - } else { - /* run various post processing (shading and geometric transformation) */ - /* FIXME: SHADER assumes 32bpp */ - /* projectm patch: removed cpu based shader - if (scope->priv->shader && - GST_VIDEO_INFO_COMP_PSTRIDE (&scope->vinfo, 0) == 4) { - scope->priv->shader (scope, &outframe, &scope->priv->tempframe); - }*/ } } gst_video_frame_unmap(&outframe); @@ -842,8 +780,9 @@ not_negotiated: { } } -static gboolean gst_audio_visualizer_src_event(GstPad *pad, GstObject *parent, - GstEvent *event) { +static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, + GstObject *parent, + GstEvent *event) { gboolean res; GstPMAudioVisualizer *scope; @@ -885,8 +824,9 @@ static gboolean gst_audio_visualizer_src_event(GstPad *pad, GstObject *parent, return res; } -static gboolean gst_audio_visualizer_sink_event(GstPad *pad, GstObject *parent, - GstEvent *event) { +static gboolean gst_pm_audio_visualizer_sink_event(GstPad *pad, + GstObject *parent, + GstEvent *event) { gboolean res; GstPMAudioVisualizer *scope; @@ -897,7 +837,7 @@ static gboolean gst_audio_visualizer_sink_event(GstPad *pad, GstObject *parent, GstCaps *caps; gst_event_parse_caps(event, &caps); - res = gst_audio_visualizer_sink_setcaps(scope, caps); + res = gst_pm_audio_visualizer_sink_setcaps(scope, caps); gst_event_unref(event); break; } @@ -922,8 +862,9 @@ static gboolean gst_audio_visualizer_sink_event(GstPad *pad, GstObject *parent, return res; } -static gboolean gst_audio_visualizer_src_query(GstPad *pad, GstObject *parent, - GstQuery *query) { +static gboolean gst_pm_audio_visualizer_src_query(GstPad *pad, + GstObject *parent, + GstQuery *query) { gboolean res = FALSE; GstPMAudioVisualizer *scope; @@ -980,8 +921,8 @@ static gboolean gst_audio_visualizer_src_query(GstPad *pad, GstObject *parent, } static GstStateChangeReturn -gst_audio_visualizer_change_state(GstElement *element, - GstStateChange transition) { +gst_pm_audio_visualizer_change_state(GstElement *element, + GstStateChange transition) { GstStateChangeReturn ret; GstPMAudioVisualizer *scope; diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index a892069..1fd9992 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -31,13 +31,9 @@ #ifndef __GST_PM_AUDIO_VISUALIZER_H__ #define __GST_PM_AUDIO_VISUALIZER_H__ -#include -#include #include #include -#include -#include #include G_BEGIN_DECLS @@ -74,6 +70,8 @@ struct _GstPMAudioVisualizer { guint64 running_time; /*< private >*/ + gpointer _padding[GST_PADDING]; + GstPMAudioVisualizerPrivate *priv; }; @@ -89,21 +87,22 @@ struct _GstPMAudioVisualizerClass { gboolean (*render)(GstPMAudioVisualizer *scope, GstBuffer *audio, GstVideoFrame *video); - /* virtual function for gl buffer pool allocation */ + /* virtual function for buffer pool allocation */ gboolean (*decide_allocation)(GstPMAudioVisualizer *scope, GstQuery *query); /* virtual function for output buffer allocation */ GstFlowReturn (*prepare_output_buffer)(GstPMAudioVisualizer *scope, GstBuffer **outbuf); + + /* virtual function for mapping the output buffer to video frame */ + void (*map_output_buffer)(GstPMAudioVisualizer *scope, + GstVideoFrame *outframe, GstBuffer *outbuf); }; -GST_PBUTILS_API GType gst_pm_audio_visualizer_get_type(void); -GST_GL_API -GstFlowReturn -gst_pm_audio_visualizer_prepare_output_buffer(GstPMAudioVisualizer *scope, - GstBuffer **outbuf); +GstFlowReturn gst_pm_audio_visualizer_default_prepare_output_buffer( + GstPMAudioVisualizer *scope, GstBuffer **outbuf); G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstPMAudioVisualizer, gst_object_unref) diff --git a/src/plugin.c b/src/plugin.c index c39440b..69be7c9 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -299,15 +299,15 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glBindTexture(GL_TEXTURE_2D, plugin->priv->textureID); /* allocate texture using glTexImage2D */ - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, - GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, - GL_UNSIGNED_BYTE, NULL); - - /* could use immutable texture buffer, but it's GL4 / ES3 and not supported on mac os x */ - //glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, - // GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - // GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, + GL_UNSIGNED_BYTE, NULL); + + /* could use immutable texture buffer, but it's GL4 / ES3 and not supported on + * mac os x */ + // glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, + // GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + // GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); From 5f45df867de5e05e7a5ae622ef17ad498e8c351c Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 22 May 2025 02:04:12 -0500 Subject: [PATCH 10/37] cleanup --- src/plugin.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 69be7c9..65aafdb 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -26,7 +26,7 @@ struct _GstProjectMPrivate { projectm_handle handle; GstGLFramebuffer *fbo; - GLuint textureID; + GLuint texture_id; GstBuffer *in_audio; GstGLMemory *mem; GstGLVideoAllocationParams *allocation_params; @@ -55,7 +55,7 @@ static GstBuffer *wrap_gl_texture(GstGLBaseAudioVisualizer *glav, return NULL; } - wrapped[0] = (gpointer)plugin->priv->textureID; + wrapped[0] = (gpointer)plugin->priv->texture_id; formats[0] = GST_GL_RGBA8; // * Wrap the texture into GLMemory. * @@ -245,7 +245,7 @@ static void gst_projectm_init(GstProjectM *plugin) { plugin->preset_locked = DEFAULT_PRESET_LOCKED; plugin->priv->handle = NULL; plugin->priv->fbo = NULL; - plugin->priv->textureID = 0; + plugin->priv->texture_id = 0; plugin->priv->in_audio = NULL; plugin->priv->mem = NULL; plugin->priv->allocation_params = NULL; @@ -270,9 +270,9 @@ static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { plugin->priv->fbo = NULL; } - if (plugin->priv->textureID) { - glDeleteTextures(1, &plugin->priv->textureID); - plugin->priv->textureID = 0; + if (plugin->priv->texture_id) { + glDeleteTextures(1, &plugin->priv->texture_id); + plugin->priv->texture_id = 0; } if (plugin->priv->allocation_params) { @@ -295,8 +295,8 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { } #endif - glGenTextures(1, &plugin->priv->textureID); - glBindTexture(GL_TEXTURE_2D, plugin->priv->textureID); + glGenTextures(1, &plugin->priv->texture_id); + glBindTexture(GL_TEXTURE_2D, plugin->priv->texture_id); /* allocate texture using glTexImage2D */ glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), @@ -319,7 +319,7 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { plugin->priv->allocation_params = gst_gl_video_allocation_params_new_wrapped_texture( glav->context, NULL, &gstav->vinfo, 0, NULL, GST_GL_TEXTURE_TARGET_2D, - GST_GL_RGBA, plugin->priv->textureID, NULL, 0); + GST_GL_RGBA, plugin->priv->texture_id, NULL, 0); // Check if ProjectM instance exists, and create if not if (!plugin->priv->handle) { From b838176db766178d7fdf4edf7a55227086f19b52 Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 22 May 2025 03:16:18 -0500 Subject: [PATCH 11/37] remove req_spf calculation --- src/plugin.c | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 65aafdb..d8df499 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -341,31 +341,6 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { } static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { - GstPMAudioVisualizer *bscope = GST_PM_AUDIO_VISUALIZER(glav); - GstProjectM *plugin = GST_PROJECTM(glav); - - // Calculate depth based on pixel stride and bits - gint depth = bscope->vinfo.finfo->pixel_stride[0] * - ((bscope->vinfo.finfo->bits >= 8) ? 8 : 1); - - // Calculate required samples per frame - bscope->req_spf = - (bscope->ainfo.channels * bscope->ainfo.rate * 2) / bscope->vinfo.fps_n; - - // Log audio info - GST_DEBUG_OBJECT( - glav, "Audio Information ", - bscope->ainfo.channels, bscope->ainfo.rate, - bscope->ainfo.finfo->description); - - // Log video info - GST_DEBUG_OBJECT(glav, - "Video Information ", - GST_VIDEO_INFO_WIDTH(&bscope->vinfo), - GST_VIDEO_INFO_HEIGHT(&bscope->vinfo), bscope->vinfo.fps_n, - bscope->vinfo.fps_d, depth, bscope->req_spf); - return TRUE; } From a06c1b40ae5abb93cfe98c8d41407d8019e17af5 Mon Sep 17 00:00:00 2001 From: hack Date: Fri, 23 May 2025 13:32:39 -0500 Subject: [PATCH 12/37] docs --- src/gstglbaseaudiovisualizer.h | 7 ++++--- src/gstpmaudiovisualizer.c | 15 ++++----------- src/gstpmaudiovisualizer.h | 19 ++++++++++++++++--- src/plugin.c | 9 +-------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index dacdc5d..8f0a60c 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -93,8 +93,9 @@ struct _GstGLBaseAudioVisualizer { * GstPMAudioVisualizer.setup) * * The base class for OpenGL based audio visualizers. - * - */ + * Extends GstPMAudioVisualizer to add GL rendering callbacks. + * Handles GL context and render buffers. +*/ struct _GstGLBaseAudioVisualizerClass { GstPMAudioVisualizerClass parent_class; @@ -104,7 +105,7 @@ struct _GstGLBaseAudioVisualizerClass { gboolean (*gl_start)(GstGLBaseAudioVisualizer *glav); /* called when gl context stops */ void (*gl_stop)(GstGLBaseAudioVisualizer *glav); - /* called once for the pipeline in the beginning */ + /* called when caps have been set for the pipeline */ gboolean (*setup)(GstGLBaseAudioVisualizer *glav); /* called to render each frame */ gboolean (*fill_gl_memory)(GstGLBaseAudioVisualizer *glav, diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 851851e..f48544f 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -633,6 +633,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, g_mutex_lock(&scope->priv->config_lock); /* this is what we want */ + /* samples per video frame * audio bytes per frame for both channels */ sbpf = scope->req_spf * bpf; inbuf = scope->priv->inbuf; @@ -646,7 +647,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GstBuffer *outbuf; GstVideoFrame outframe; - /* get timestamp of the current adapter content (audio input timestamp) */ + /* get timestamp of the current adapter content (audio input) */ ts = gst_adapter_prev_pts(scope->priv->adapter, &dist); if (GST_CLOCK_TIME_IS_VALID(ts)) { /* convert bytes to time */ @@ -706,15 +707,6 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, if (ret != GST_FLOW_OK) break; - /* - todo: need to set any buffer meta for gl ? - gst_buffer_add_video_meta(outbuf, GST_VIDEO_FRAME_FLAG_NONE, - GST_VIDEO_FORMAT_RGBA, scope->vinfo.width, - scope->vinfo.height); - gst_video_info_set_format (&scope->vinfo, GST_VIDEO_FORMAT_RGBA, - scope->vinfo.width, scope->vinfo.height); - */ - /* sync controlled properties */ if (GST_CLOCK_TIME_IS_VALID(ts)) gst_object_sync_values(GST_OBJECT(scope), ts); @@ -726,9 +718,10 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, if (!(adata = (gpointer)gst_adapter_map(scope->priv->adapter, sbpf))) break; - // projectm patch: modification to customize mapping + /* allow customized memory to video frame mapping */ klass->map_output_buffer(scope, &outframe, outbuf); + /* place sbpf number of bytes of audio data into inbuf */ gst_buffer_replace_all_memory( inbuf, gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY, adata, sbpf, 0, sbpf, NULL, NULL)); diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index 1fd9992..5b5c22c 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -47,9 +47,9 @@ G_BEGIN_DECLS #define GST_PM_AUDIO_VISUALIZER_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_PM_AUDIO_VISUALIZER, \ GstPMAudioVisualizerClass)) -#define GST_IS_SYNAESTHESIA(obj) \ +#define GST_PM_IS_SYNAESTHESIA(obj) \ (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_PM_AUDIO_VISUALIZER)) -#define GST_IS_SYNAESTHESIA_CLASS(klass) \ +#define GST_PM_IS_SYNAESTHESIA_CLASS(klass) \ (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_PM_AUDIO_VISUALIZER)) typedef struct _GstPMAudioVisualizer GstPMAudioVisualizer; typedef struct _GstPMAudioVisualizerClass GstPMAudioVisualizerClass; @@ -58,7 +58,8 @@ typedef struct _GstPMAudioVisualizerPrivate GstPMAudioVisualizerPrivate; struct _GstPMAudioVisualizer { GstElement parent; - guint req_spf; /* min samples per frame wanted by the subclass */ + /* min samples per frame wanted by the subclass (one channel) */ + guint req_spf; /* video state */ GstVideoInfo vinfo; @@ -75,6 +76,18 @@ struct _GstPMAudioVisualizer { GstPMAudioVisualizerPrivate *priv; }; +/** + * GstPMAudioVisualizerClass: + * @decide_allocation: buffer pool allocation + * @prepare_output_buffer: allocate a buffer for rendering a frame. + * @map_output_buffer: map video frame to memory buffer. + * @render: render a frame from an audio buffer. + * @setup: called whenever the format changes. + * + * Base class for audio visualizers, derived from gstreamer + * GstAudioVisualizerClass. This plugin handles rendering video frames with a + * fixed framerate from audio input samples. + */ struct _GstPMAudioVisualizerClass { /*< private >*/ GstElementClass parent_class; diff --git a/src/plugin.c b/src/plugin.c index d8df499..ea3c7f2 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -298,17 +298,11 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glGenTextures(1, &plugin->priv->texture_id); glBindTexture(GL_TEXTURE_2D, plugin->priv->texture_id); - /* allocate texture using glTexImage2D */ + // allocate texture glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL); - /* could use immutable texture buffer, but it's GL4 / ES3 and not supported on - * mac os x */ - // glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, - // GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - // GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); @@ -344,7 +338,6 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { return TRUE; } -// TODO: CLEANUP & ADD DEBUGGING static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstProjectM *plugin = GST_PROJECTM(stuff); GstGLBaseAudioVisualizer *gstav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); From 156e97b3c4a84087e2213fb900e567fe7ae4a231 Mon Sep 17 00:00:00 2001 From: hack Date: Wed, 28 May 2025 04:03:24 -0500 Subject: [PATCH 13/37] fix timing issues --- src/gstpmaudiovisualizer.c | 22 +++++++++++++++------- src/gstpmaudiovisualizer.h | 5 ++++- src/plugin.c | 12 ++++++------ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index f48544f..794115f 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -50,13 +50,9 @@ #include -#include -#include -#include #include #include "gstpmaudiovisualizer.h" -#include #include GST_DEBUG_CATEGORY_STATIC(pm_audio_visualizer_debug); @@ -249,6 +245,8 @@ static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, scope->priv->adapter = gst_adapter_new(); scope->priv->inbuf = gst_buffer_new(); + scope->stream_time = 0; + scope->running_time = 0; /* properties */ @@ -370,10 +368,13 @@ static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, if (klass->setup && !klass->setup(scope)) goto setup_failed; - GST_DEBUG_OBJECT(scope, "video: dimension %dx%d, framerate %d/%d", + GST_INFO_OBJECT(scope, "video: dimension %dx%d, framerate %d/%d", GST_VIDEO_INFO_WIDTH(&info), GST_VIDEO_INFO_HEIGHT(&info), GST_VIDEO_INFO_FPS_N(&info), GST_VIDEO_INFO_FPS_D(&info)); - GST_DEBUG_OBJECT(scope, "blocks: spf %u, req_spf %u", scope->priv->spf, + GST_INFO_OBJECT(scope, "audio: rate %d, channels: %d, bpf: %d", + GST_AUDIO_INFO_RATE(&scope->ainfo), GST_AUDIO_INFO_CHANNELS(&scope->ainfo), + GST_AUDIO_INFO_BPF(&scope->ainfo)); + GST_INFO_OBJECT(scope, "blocks: spf %u, req_spf %u", scope->priv->spf, scope->req_spf); gst_pad_set_caps(scope->priv->srcpad, caps); @@ -695,6 +696,8 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, } } + scope->stream_time = gst_segment_to_stream_time(&scope->priv->segment, + GST_FORMAT_TIME, ts); ++scope->priv->processed; g_mutex_unlock(&scope->priv->config_lock); @@ -747,11 +750,16 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GST_LOG_OBJECT(scope, "avail: %u, bpf: %u", avail, sbpf); /* we want to take less or more, depending on spf : req_spf */ if (avail - sbpf >= sbpf) { + // more than one frame available gst_adapter_flush(scope->priv->adapter, sbpf); gst_adapter_unmap(scope->priv->adapter); } else if (avail >= sbpf) { /* just flush a bit and stop */ - gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); + // todo: this messes with the length and timing when using offline rendering. seems like a bug in the original code + //gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); + + // instead just take one frame and stop + gst_adapter_flush(scope->priv->adapter, sbpf); gst_adapter_unmap(scope->priv->adapter); break; } diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index 5b5c22c..a69e01f 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -67,9 +67,12 @@ struct _GstPMAudioVisualizer { /* audio state */ GstAudioInfo ainfo; - /* current pts running time for syncing renderers */ + /* current pts running time for syncing pipeline */ guint64 running_time; + /* current time position within the input stream. may be the best choice for syncing renderers */ + guint64 stream_time; + /*< private >*/ gpointer _padding[GST_PADDING]; diff --git a/src/plugin.c b/src/plugin.c index ea3c7f2..833fd01 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -52,7 +52,6 @@ static GstBuffer *wrap_gl_texture(GstGLBaseAudioVisualizer *glav, buffer = gst_buffer_new(); if (!buffer) { g_error("Failed to create new buffer\n"); - return NULL; } wrapped[0] = (gpointer)plugin->priv->texture_id; @@ -63,7 +62,6 @@ static GstBuffer *wrap_gl_texture(GstGLBaseAudioVisualizer *glav, allocator, buffer, plugin->priv->allocation_params, formats, wrapped, 1); if (!ret) { g_error("Failed to setup gl memory\n"); - return NULL; } gst_object_unref(allocator); @@ -77,7 +75,7 @@ gst_projectm_prepare_output_buffer(GstGLBaseAudioVisualizer *scope, GstProjectM *plugin = GST_PROJECTM(scope); *outbuf = wrap_gl_texture(scope, plugin); - GST_INFO_OBJECT(plugin, "Wrapped RT texture buffer"); + GST_DEBUG_OBJECT(plugin, "Wrapped RT texture buffer"); return GST_FLOW_OK; } @@ -341,13 +339,15 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstProjectM *plugin = GST_PROJECTM(stuff); GstGLBaseAudioVisualizer *gstav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); - GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(stuff); + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(plugin); GstMapInfo audioMap; gboolean result = TRUE; - // get current gst (PTS) time and set projectM time - gdouble seconds_since_first_frame = (gdouble)pmav->running_time / GST_SECOND; + // get current gst stream time and set projectM time + // todo: PTS or stream time ? + gdouble seconds_since_first_frame = (double)pmav->stream_time / GST_SECOND; + projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); // AUDIO From a33e7aed3123fabaa04d54f330863326800d7158 Mon Sep 17 00:00:00 2001 From: hack Date: Wed, 28 May 2025 21:41:13 -0500 Subject: [PATCH 14/37] add config PROP_PTS_SYNC to configure timestamp --- src/config.h | 1 + src/enums.h | 3 +- src/gstglbaseaudiovisualizer.c | 2 ++ src/gstglbaseaudiovisualizer.h | 5 +++- src/gstpmaudiovisualizer.c | 28 ++++++++++--------- src/gstpmaudiovisualizer.h | 9 ++---- src/plugin.c | 51 ++++++++++++++++++++++++---------- src/plugin.h | 1 + 8 files changed, 65 insertions(+), 35 deletions(-) diff --git a/src/config.h b/src/config.h index bc83e3b..ce2cc92 100644 --- a/src/config.h +++ b/src/config.h @@ -33,6 +33,7 @@ G_BEGIN_DECLS #define DEFAULT_PRESET_LOCKED FALSE #define DEFAULT_ENABLE_PLAYLIST TRUE #define DEFAULT_SHUFFLE_PRESETS TRUE // depends on ENABLE_PLAYLIST +#define DEFAULT_PTS_SYNC TRUE G_END_DECLS diff --git a/src/enums.h b/src/enums.h index 863d677..c3b5969 100644 --- a/src/enums.h +++ b/src/enums.h @@ -24,7 +24,8 @@ enum { PROP_EASTER_EGG, PROP_PRESET_LOCKED, PROP_SHUFFLE_PRESETS, - PROP_ENABLE_PLAYLIST + PROP_ENABLE_PLAYLIST, + PROP_PTS_SYNC }; G_END_DECLS diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 2b4f53e..4e21525 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -178,6 +178,7 @@ static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv->in_audio = NULL; glav->priv->out_tex = NULL; glav->context = NULL; + glav->pts = 0; glav->priv->timestamp_offset = 0; g_rec_mutex_init(&glav->priv->context_lock); gst_gl_base_audio_visualizer_start(glav); @@ -358,6 +359,7 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, GstBuffer *buffer = video->buffer; + glav->pts = GST_BUFFER_PTS(buffer); glav->priv->in_audio = audio; gst_gl_context_thread_add(glav->context, (GstGLContextThreadFunc)_fill_gl, diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index 8f0a60c..8d1e110 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -77,6 +77,9 @@ struct _GstGLBaseAudioVisualizer { GstGLDisplay *display; GstGLContext *context; + /* current buffer presentation timestamp */ + guint64 pts; + /*< private >*/ gpointer _padding[GST_PADDING]; @@ -95,7 +98,7 @@ struct _GstGLBaseAudioVisualizer { * The base class for OpenGL based audio visualizers. * Extends GstPMAudioVisualizer to add GL rendering callbacks. * Handles GL context and render buffers. -*/ + */ struct _GstGLBaseAudioVisualizerClass { GstPMAudioVisualizerClass parent_class; diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 794115f..6c53aa1 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -246,7 +246,6 @@ static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, scope->priv->adapter = gst_adapter_new(); scope->priv->inbuf = gst_buffer_new(); scope->stream_time = 0; - scope->running_time = 0; /* properties */ @@ -369,13 +368,14 @@ static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, goto setup_failed; GST_INFO_OBJECT(scope, "video: dimension %dx%d, framerate %d/%d", - GST_VIDEO_INFO_WIDTH(&info), GST_VIDEO_INFO_HEIGHT(&info), - GST_VIDEO_INFO_FPS_N(&info), GST_VIDEO_INFO_FPS_D(&info)); + GST_VIDEO_INFO_WIDTH(&info), GST_VIDEO_INFO_HEIGHT(&info), + GST_VIDEO_INFO_FPS_N(&info), GST_VIDEO_INFO_FPS_D(&info)); GST_INFO_OBJECT(scope, "audio: rate %d, channels: %d, bpf: %d", - GST_AUDIO_INFO_RATE(&scope->ainfo), GST_AUDIO_INFO_CHANNELS(&scope->ainfo), - GST_AUDIO_INFO_BPF(&scope->ainfo)); + GST_AUDIO_INFO_RATE(&scope->ainfo), + GST_AUDIO_INFO_CHANNELS(&scope->ainfo), + GST_AUDIO_INFO_BPF(&scope->ainfo)); GST_INFO_OBJECT(scope, "blocks: spf %u, req_spf %u", scope->priv->spf, - scope->req_spf); + scope->req_spf); gst_pad_set_caps(scope->priv->srcpad, caps); @@ -660,11 +660,12 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GstClockTime earliest_time; gdouble proportion; guint64 qostime; + guint64 running_time; - scope->running_time = gst_segment_to_running_time(&scope->priv->segment, - GST_FORMAT_TIME, ts); + running_time = gst_segment_to_running_time(&scope->priv->segment, + GST_FORMAT_TIME, ts); - qostime = scope->running_time + scope->priv->frame_duration; + qostime = running_time + scope->priv->frame_duration; GST_OBJECT_LOCK(scope); earliest_time = scope->priv->earliest_time; @@ -696,8 +697,8 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, } } - scope->stream_time = gst_segment_to_stream_time(&scope->priv->segment, - GST_FORMAT_TIME, ts); + scope->stream_time = + gst_segment_to_stream_time(&scope->priv->segment, GST_FORMAT_TIME, ts); ++scope->priv->processed; g_mutex_unlock(&scope->priv->config_lock); @@ -755,8 +756,9 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, gst_adapter_unmap(scope->priv->adapter); } else if (avail >= sbpf) { /* just flush a bit and stop */ - // todo: this messes with the length and timing when using offline rendering. seems like a bug in the original code - //gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); + // todo: this messes with the length and timing when using offline + // rendering. seems like a bug in the original code + // gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); // instead just take one frame and stop gst_adapter_flush(scope->priv->adapter, sbpf); diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index a69e01f..2022c28 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -47,9 +47,9 @@ G_BEGIN_DECLS #define GST_PM_AUDIO_VISUALIZER_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_PM_AUDIO_VISUALIZER, \ GstPMAudioVisualizerClass)) -#define GST_PM_IS_SYNAESTHESIA(obj) \ +#define GST_PM_IS_SYNAESTHESIA(obj) \ (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_PM_AUDIO_VISUALIZER)) -#define GST_PM_IS_SYNAESTHESIA_CLASS(klass) \ +#define GST_PM_IS_SYNAESTHESIA_CLASS(klass) \ (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_PM_AUDIO_VISUALIZER)) typedef struct _GstPMAudioVisualizer GstPMAudioVisualizer; typedef struct _GstPMAudioVisualizerClass GstPMAudioVisualizerClass; @@ -67,10 +67,7 @@ struct _GstPMAudioVisualizer { /* audio state */ GstAudioInfo ainfo; - /* current pts running time for syncing pipeline */ - guint64 running_time; - - /* current time position within the input stream. may be the best choice for syncing renderers */ + /* current time (ns) position within the input stream */ guint64 stream_time; /*< private >*/ diff --git a/src/plugin.c b/src/plugin.c index 833fd01..4708a5a 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -42,31 +42,34 @@ G_DEFINE_TYPE_WITH_CODE(GstProjectM, gst_projectm, static GstBuffer *wrap_gl_texture(GstGLBaseAudioVisualizer *glav, GstProjectM *plugin) { GstGLMemoryAllocator *allocator; - gpointer wrapped[1]; - GstGLFormat formats[1]; - GstBuffer *buffer; + gpointer glTextures[1]; + GstGLFormat glFormats[1]; + GstBuffer *glBuffer; gboolean ret; allocator = gst_gl_memory_allocator_get_default(glav->context); - buffer = gst_buffer_new(); - if (!buffer) { + glBuffer = gst_buffer_new(); + if (!glBuffer) { g_error("Failed to create new buffer\n"); + return NULL; } - wrapped[0] = (gpointer)plugin->priv->texture_id; - formats[0] = GST_GL_RGBA8; + glTextures[0] = (gpointer)plugin->priv->texture_id; + glFormats[0] = GST_GL_RGBA8; - // * Wrap the texture into GLMemory. * - ret = gst_gl_memory_setup_buffer( - allocator, buffer, plugin->priv->allocation_params, formats, wrapped, 1); + // create gl mem buffer for texture + ret = gst_gl_memory_setup_buffer(allocator, glBuffer, + plugin->priv->allocation_params, glFormats, + glTextures, 1); if (!ret) { g_error("Failed to setup gl memory\n"); + return NULL; } gst_object_unref(allocator); - return buffer; + return glBuffer; } static GstFlowReturn @@ -142,6 +145,8 @@ void gst_projectm_set_property(GObject *object, guint property_id, case PROP_SHUFFLE_PRESETS: plugin->shuffle_presets = g_value_get_boolean(value); break; + case PROP_PTS_SYNC: + plugin->pts_sync = g_value_get_boolean(value); default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -202,6 +207,9 @@ void gst_projectm_get_property(GObject *object, guint property_id, case PROP_SHUFFLE_PRESETS: g_value_set_boolean(value, plugin->shuffle_presets); break; + case PROP_PTS_SYNC: + g_value_set_boolean(value, plugin->pts_sync); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -222,6 +230,7 @@ static void gst_projectm_init(GstProjectM *plugin) { plugin->preset_duration = DEFAULT_PRESET_DURATION; plugin->enable_playlist = DEFAULT_ENABLE_PLAYLIST; plugin->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; + plugin->pts_sync = true; const gchar *meshSizeStr = DEFAULT_MESH_SIZE; gint width, height; @@ -344,9 +353,15 @@ static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstMapInfo audioMap; gboolean result = TRUE; - // get current gst stream time and set projectM time - // todo: PTS or stream time ? - gdouble seconds_since_first_frame = (double)pmav->stream_time / GST_SECOND; + // get current gst pts or stream time (dts) and set projectM time + gdouble seconds_since_first_frame; + if (plugin->pts_sync) { + // sync to pts + seconds_since_first_frame = (double)gstav->pts / GST_SECOND; + } else { + // sync to dts + seconds_since_first_frame = (double)pmav->stream_time / GST_SECOND; + } projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); @@ -550,6 +565,14 @@ static void gst_projectm_class_init(GstProjectMClass *klass) { "and not locked. Playlist must be enabled for this to take effect.", DEFAULT_SHUFFLE_PRESETS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property( + gobject_class, PROP_PTS_SYNC, + g_param_spec_boolean( + "pts-sync", "Presentation Timestamp Sync", + "If true, projectM will be synced to the gst presentation timestamp. " + "In case of false, the stream time (dts) will be used.", + DEFAULT_PTS_SYNC, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + gobject_class->finalize = gst_projectm_finalize; scope_class->supported_gl_api = GST_GL_API_OPENGL3 | GST_GL_API_GLES2; diff --git a/src/plugin.h b/src/plugin.h index 4210fd0..da59250 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -36,6 +36,7 @@ struct _GstProjectM { gboolean preset_locked; gboolean enable_playlist; gboolean shuffle_presets; + gboolean pts_sync; GstProjectMPrivate *priv; }; From 94a970cecb36bb384e968e6c08697e7b729ad77b Mon Sep 17 00:00:00 2001 From: hack Date: Wed, 28 May 2025 23:03:54 -0500 Subject: [PATCH 15/37] restore relative frame time calc --- src/plugin.c | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 4708a5a..4d2f1be 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -25,6 +25,9 @@ GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); struct _GstProjectMPrivate { projectm_handle handle; + GstClockTime first_frame_time; + gboolean first_frame_received; + GstGLFramebuffer *fbo; GLuint texture_id; GstBuffer *in_audio; @@ -231,6 +234,8 @@ static void gst_projectm_init(GstProjectM *plugin) { plugin->enable_playlist = DEFAULT_ENABLE_PLAYLIST; plugin->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; plugin->pts_sync = true; + plugin->priv->first_frame_time = 0; + plugin->priv->first_frame_received = FALSE; const gchar *meshSizeStr = DEFAULT_MESH_SIZE; gint width, height; @@ -326,6 +331,7 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { if (!plugin->priv->handle) { // Create ProjectM instance plugin->priv->handle = projectm_init(plugin); + plugin->priv->first_frame_received = FALSE; if (!plugin->priv->handle) { GST_ERROR_OBJECT(plugin, "ProjectM could not be initialized"); return FALSE; @@ -345,24 +351,44 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { return TRUE; } +static gdouble get_seconds_since_first_frame(GstProjectM *plugin, GstGLBaseAudioVisualizer *gstav) +{ + // pick timestamp to sync to + GstClockTime current_time; + if (plugin->pts_sync) { + // sync to pts + current_time = gstav->pts; + } else { + // sync to dts + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(plugin); + current_time = pmav->stream_time; + } + + if (!plugin->priv->first_frame_received) { + // Store the timestamp of the first frame + plugin->priv->first_frame_time = current_time; + plugin->priv->first_frame_received = TRUE; + return 0.0; + } + + // Calculate elapsed time + GstClockTime elapsed_time = current_time - plugin->priv->first_frame_time; + + // Convert to fractional seconds + gdouble elapsed_seconds = (gdouble) elapsed_time / GST_SECOND; + + return elapsed_seconds; +} + static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstProjectM *plugin = GST_PROJECTM(stuff); GstGLBaseAudioVisualizer *gstav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); - GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(plugin); GstMapInfo audioMap; gboolean result = TRUE; // get current gst pts or stream time (dts) and set projectM time - gdouble seconds_since_first_frame; - if (plugin->pts_sync) { - // sync to pts - seconds_since_first_frame = (double)gstav->pts / GST_SECOND; - } else { - // sync to dts - seconds_since_first_frame = (double)pmav->stream_time / GST_SECOND; - } - + gdouble seconds_since_first_frame = get_seconds_since_first_frame(plugin, gstav); projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); // AUDIO From fde0fddbd449905d0d2501039331a682a9a82f2e Mon Sep 17 00:00:00 2001 From: hack Date: Wed, 28 May 2025 23:28:46 -0500 Subject: [PATCH 16/37] clang-format --- src/plugin.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 4d2f1be..589f983 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -351,8 +351,8 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { return TRUE; } -static gdouble get_seconds_since_first_frame(GstProjectM *plugin, GstGLBaseAudioVisualizer *gstav) -{ +static gdouble get_seconds_since_first_frame(GstProjectM *plugin, + GstGLBaseAudioVisualizer *gstav) { // pick timestamp to sync to GstClockTime current_time; if (plugin->pts_sync) { @@ -375,7 +375,7 @@ static gdouble get_seconds_since_first_frame(GstProjectM *plugin, GstGLBaseAudio GstClockTime elapsed_time = current_time - plugin->priv->first_frame_time; // Convert to fractional seconds - gdouble elapsed_seconds = (gdouble) elapsed_time / GST_SECOND; + gdouble elapsed_seconds = (gdouble)elapsed_time / GST_SECOND; return elapsed_seconds; } @@ -388,7 +388,9 @@ static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { gboolean result = TRUE; // get current gst pts or stream time (dts) and set projectM time - gdouble seconds_since_first_frame = get_seconds_since_first_frame(plugin, gstav); + gdouble seconds_since_first_frame = + get_seconds_since_first_frame(plugin, gstav); + projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); // AUDIO From ed4ffc83f9bbc5d9f66c5705350b739859b4b980 Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 29 May 2025 22:06:40 -0500 Subject: [PATCH 17/37] naming, additional docs --- src/gstglbaseaudiovisualizer.c | 81 +++++++++++++++++++++++++--------- src/gstpmaudiovisualizer.c | 26 ++++++++--- src/gstpmaudiovisualizer.h | 5 ++- src/plugin.c | 12 ++--- 4 files changed, 90 insertions(+), 34 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 4e21525..36d1239 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -50,8 +50,9 @@ * #GstGLBaseAudioVisualizer handles the nitty gritty details of retrieving an * OpenGL context. It also provides `gl_start()` and `gl_stop()` virtual methods * that ensure an OpenGL context is available and current in the calling thread - * for initializing and cleaning up OpenGL dependent resources. The `gl_render` - * virtual method is used to perform OpenGL rendering. + * for initializing and cleaning up OpenGL dependent resources. The `render` + * virtual method of the GstPMAudioVisualizer is implemented to perform OpenGL + * rendering. fill_gl_memory is called to render directly to gl memory. */ #define GST_CAT_DEFAULT gst_gl_base_audio_visualizer_debug @@ -95,42 +96,69 @@ static void gst_gl_base_audio_visualizer_get_property(GObject *object, GValue *value, GParamSpec *pspec); +/* discover gl context / display from gst */ static void gst_gl_base_audio_visualizer_set_context(GstElement *element, GstContext *context); +/* handle pipeline state changes */ static GstStateChangeReturn gst_gl_base_audio_visualizer_change_state(GstElement *element, GstStateChange transition); -static gboolean -gst_gl_base_audio_visualizer_render(GstPMAudioVisualizer *bscope, - GstBuffer *audio, GstVideoFrame *video); +/* render a video frame frame */ +static gboolean gst_gl_base_audio_visualizer_parent_render( + GstPMAudioVisualizer *bscope, GstBuffer *audio, GstVideoFrame *video); + +/* internal utility for resetting state on start */ static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav); + +/* internal utility for cleaning up gl context on stop */ static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav); -static gboolean -gst_gl_base_audio_visualizer_decide_allocation(GstPMAudioVisualizer *gstav, - GstQuery *query); +/* gl memory pool allocation impl for parent class GstPMAudioVisualizerClass */ +static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( + GstPMAudioVisualizer *gstav, GstQuery *query); + +/* called when format changes, default v-impl for this class. can be overwritten + * by implementer. */ static gboolean gst_gl_base_audio_visualizer_default_setup(GstGLBaseAudioVisualizer *glav); + +/* gl context is started, default v-impl for this class. can be overwritten + * by implementer. */ static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav); + +/* gl context is shutting down, default v-impl for this class. can be + * overwritten by implementer. */ static void gst_gl_base_audio_visualizer_default_gl_stop(GstGLBaseAudioVisualizer *glav); + +/* default empty v-impl for rendering a frame. can be overwritten by + * implementer. */ static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( GstGLBaseAudioVisualizer *glav, GstBuffer *in_audio, GstGLMemory *mem); +/* find a valid gl context. lock must have already been acquired. */ static gboolean gst_gl_base_audio_visualizer_find_gl_context_unlocked( GstGLBaseAudioVisualizer *glav); -static gboolean gst_gl_base_audio_visualizer_setup(GstPMAudioVisualizer *gstav); +/* called whenever the format changes, impl for parent class + * GstPMAudioVisualizerClass */ +static gboolean +gst_gl_base_audio_visualizer_parent_setup(GstPMAudioVisualizer *gstav); +/* output buffer allocation default v-impl for this class. can be overwritten by + * implementer. */ static GstFlowReturn gst_gl_base_audio_visualizer_default_prepare_output_buffer( GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf); +/* output buffer allocation impl for parent class GstPMAudioVisualizerClass */ static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( GstPMAudioVisualizer *scope, GstBuffer **outbuf); -static void gst_gl_base_audio_visualizer_map_output_buffer( +/* map output video frame to buffer outbuf with gl flags, impl for parent class + * GstPMAudioVisualizerClass */ +static void gst_gl_base_audio_visualizer_parent_map_output_buffer( GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf); static void @@ -150,23 +178,33 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_change_state); gstav_class->decide_allocation = - GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_decide_allocation); - gstav_class->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_setup); + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_decide_allocation); + + gstav_class->setup = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_setup); + + gstav_class->render = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_render); - gstav_class->render = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_render); gstav_class->prepare_output_buffer = GST_DEBUG_FUNCPTR( gst_gl_base_audio_visualizer_parent_prepare_output_buffer); + gstav_class->map_output_buffer = - GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_map_output_buffer); + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_map_output_buffer); klass->supported_gl_api = GST_GL_API_ANY; + klass->gl_start = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_start); + klass->gl_stop = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_stop); + klass->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_setup); + klass->fill_gl_memory = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_fill_gl_memory); + klass->prepare_output_buffer = GST_DEBUG_FUNCPTR( gst_gl_base_audio_visualizer_default_prepare_output_buffer); } @@ -315,7 +353,7 @@ static GstFlowReturn gst_gl_base_audio_visualizer_parent_prepare_output_buffer( return klass->prepare_output_buffer(glav, outbuf); } -static void gst_gl_base_audio_visualizer_map_output_buffer( +static void gst_gl_base_audio_visualizer_parent_map_output_buffer( GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf) { /* map video to gl memory */ gst_video_frame_map(outframe, &scope->vinfo, outbuf, @@ -362,6 +400,7 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, glav->pts = GST_BUFFER_PTS(buffer); glav->priv->in_audio = audio; + // dispatch _fill_gl to the gl thread gst_gl_context_thread_add(glav->context, (GstGLContextThreadFunc)_fill_gl, glav); @@ -418,7 +457,7 @@ eos: { } static gboolean -gst_gl_base_audio_visualizer_setup(GstPMAudioVisualizer *gstav) { +gst_gl_base_audio_visualizer_parent_setup(GstPMAudioVisualizer *gstav) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); GstGLBaseAudioVisualizerClass *glav_class = GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(gstav); @@ -428,9 +467,8 @@ gst_gl_base_audio_visualizer_setup(GstPMAudioVisualizer *gstav) { return glav_class->setup(glav); } -static gboolean -gst_gl_base_audio_visualizer_render(GstPMAudioVisualizer *bscope, - GstBuffer *audio, GstVideoFrame *video) { +static gboolean gst_gl_base_audio_visualizer_parent_render( + GstPMAudioVisualizer *bscope, GstBuffer *audio, GstVideoFrame *video) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(bscope); gst_gl_base_audio_visualizer_fill(bscope, glav, audio, video); @@ -593,9 +631,8 @@ error: { } } -static gboolean -gst_gl_base_audio_visualizer_decide_allocation(GstPMAudioVisualizer *gstav, - GstQuery *query) { +static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( + GstPMAudioVisualizer *gstav, GstQuery *query) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); GstGLContext *context; GstBufferPool *pool = NULL; diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 6c53aa1..c785cc1 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -35,13 +35,24 @@ /* * The code in this file is based on - * GStreamer / gst-plugins-base / 1.19.2: + * GStreamer / gst-plugins-base / 1.19.2, latest version as of 2025/05/29. * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h * Original copyright notice has been retained at the top of this file. * - * The code has been modified to map gl memory for the video output buffer and - * expose pts running_time. Support for CPU based shaders has been removed. + * The code has been modified to improve compatibility with projectM and OpenGL. + * + * - Adds apis for implementer-provided memory allocation and video output + * buffer mapping. Useful for directly mapping GL memory. + * + * - Expose the stream time (dts) state. + * + * - Main memory buffers have been removed. + * + * - Cpu based transition shaders have been removed. + * + * - Bugfix for the amount of bytes that are flushed for a single frame from + * audio buffers. */ #ifdef HAVE_CONFIG_H @@ -634,6 +645,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, g_mutex_lock(&scope->priv->config_lock); /* this is what we want */ + /* number of audio bytes to process for one video frame */ /* samples per video frame * audio bytes per frame for both channels */ sbpf = scope->req_spf * bpf; @@ -697,8 +709,10 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, } } + // get stream time for others interested in timing information scope->stream_time = gst_segment_to_stream_time(&scope->priv->segment, GST_FORMAT_TIME, ts); + ++scope->priv->processed; g_mutex_unlock(&scope->priv->config_lock); @@ -751,16 +765,18 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GST_LOG_OBJECT(scope, "avail: %u, bpf: %u", avail, sbpf); /* we want to take less or more, depending on spf : req_spf */ if (avail - sbpf >= sbpf) { - // more than one frame available + // enough audio data for more frames is available gst_adapter_flush(scope->priv->adapter, sbpf); gst_adapter_unmap(scope->priv->adapter); } else if (avail >= sbpf) { + // was just enough audio data for one frame /* just flush a bit and stop */ // todo: this messes with the length and timing when using offline // rendering. seems like a bug in the original code // gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); - // instead just take one frame and stop + // instead just flush one video frame worth of audio data from the buffer + // and stop gst_adapter_flush(scope->priv->adapter, sbpf); gst_adapter_unmap(scope->priv->adapter); break; diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index 2022c28..36fcaa8 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -22,10 +22,13 @@ /* * The code in this file is based on - * GStreamer / gst-plugins-base / 1.19.2: + * GStreamer / gst-plugins-base / 1.19.2, latest version as of 2025/05/29. * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h + * * Original copyright notice has been retained at the top of this file. + * The code has been modified to improve compatibility with projectM and OpenGL. + * See impl for details. */ #ifndef __GST_PM_AUDIO_VISUALIZER_H__ diff --git a/src/plugin.c b/src/plugin.c index 589f983..87943a1 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -352,12 +352,12 @@ static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { } static gdouble get_seconds_since_first_frame(GstProjectM *plugin, - GstGLBaseAudioVisualizer *gstav) { + GstGLBaseAudioVisualizer *glav) { // pick timestamp to sync to GstClockTime current_time; if (plugin->pts_sync) { // sync to pts - current_time = gstav->pts; + current_time = glav->pts; } else { // sync to dts GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(plugin); @@ -382,14 +382,14 @@ static gdouble get_seconds_since_first_frame(GstProjectM *plugin, static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { GstProjectM *plugin = GST_PROJECTM(stuff); - GstGLBaseAudioVisualizer *gstav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(stuff); GstMapInfo audioMap; gboolean result = TRUE; - // get current gst pts or stream time (dts) and set projectM time + // get current gst sync time (pts or stream time/dts) and set projectM time gdouble seconds_since_first_frame = - get_seconds_since_first_frame(plugin, gstav); + get_seconds_since_first_frame(plugin, glav); projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); @@ -414,7 +414,7 @@ static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { projectm_opengl_render_frame_fbo(plugin->priv->handle, plugin->priv->fbo->fbo_id); - gl_error_handler(gstav->context, plugin); + gl_error_handler(glav->context, plugin); gst_buffer_unmap(plugin->priv->in_audio, &audioMap); From aaaaac37587138919dced2dd60d86d3833206277 Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 29 May 2025 23:42:13 -0500 Subject: [PATCH 18/37] fix typo --- src/gstglbaseaudiovisualizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 36d1239..107a36a 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -104,7 +104,7 @@ static GstStateChangeReturn gst_gl_base_audio_visualizer_change_state(GstElement *element, GstStateChange transition); -/* render a video frame frame */ +/* renders a video frame using gl, impl for parent class GstPMAudioVisualizerClass. */ static gboolean gst_gl_base_audio_visualizer_parent_render( GstPMAudioVisualizer *bscope, GstBuffer *audio, GstVideoFrame *video); From 881150bae9426ea08b812c6a49dd1d41598ee9dc Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 29 May 2025 23:42:39 -0500 Subject: [PATCH 19/37] fix typo --- src/gstglbaseaudiovisualizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 107a36a..805a017 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -104,7 +104,8 @@ static GstStateChangeReturn gst_gl_base_audio_visualizer_change_state(GstElement *element, GstStateChange transition); -/* renders a video frame using gl, impl for parent class GstPMAudioVisualizerClass. */ +/* renders a video frame using gl, impl for parent class + * GstPMAudioVisualizerClass. */ static gboolean gst_gl_base_audio_visualizer_parent_render( GstPMAudioVisualizer *bscope, GstBuffer *audio, GstVideoFrame *video); From 1df48150b7af4ebb500e66046abb256b8155f888 Mon Sep 17 00:00:00 2001 From: hack Date: Fri, 30 May 2025 00:26:37 -0500 Subject: [PATCH 20/37] fix typo, restore debug logs --- src/gstpmaudiovisualizer.c | 10 +++++----- src/plugin.c | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index c785cc1..cc9caff 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -42,17 +42,17 @@ * * The code has been modified to improve compatibility with projectM and OpenGL. * - * - Adds apis for implementer-provided memory allocation and video output - * buffer mapping. Useful for directly mapping GL memory. + * - New apis for implementer-provided memory allocation and video frame + * buffer mapping. Useful for mapping video frames directly to gl memory. * * - Expose the stream time (dts) state. * - * - Main memory buffers have been removed. + * - Main memory based video frame buffers have been removed. * * - Cpu based transition shaders have been removed. * - * - Bugfix for the amount of bytes that are flushed for a single frame from - * audio buffers. + * - Bugfix for the amount of bytes being flushed for a single video frame from + * the audio input buffer. */ #ifdef HAVE_CONFIG_H diff --git a/src/plugin.c b/src/plugin.c index 87943a1..748bc9b 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -348,6 +348,22 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { } static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { + + GstPMAudioVisualizer *gstav = GST_PM_AUDIO_VISUALIZER(glav); + + // Log audio info + GST_DEBUG_OBJECT( + glav, "Audio Information ", + gstav->ainfo.channels, gstav->ainfo.rate, + gstav->ainfo.finfo->description); + + // Log video info + GST_DEBUG_OBJECT( + glav, + "Video Information ", + GST_VIDEO_INFO_WIDTH(&gstav->vinfo), GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), + gstav->vinfo.fps_n, gstav->vinfo.fps_d, gstav->req_spf); + return TRUE; } From 35ce12f1c77519e60dc34454e7bde218436da6b3 Mon Sep 17 00:00:00 2001 From: hack Date: Sun, 1 Jun 2025 23:21:21 -0500 Subject: [PATCH 21/37] use gst vtable for gl functions --- src/plugin.c | 20 ++++++++++++-------- src/plugin.h | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index 748bc9b..9009818 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -307,20 +307,24 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { } #endif - glGenTextures(1, &plugin->priv->texture_id); - glBindTexture(GL_TEXTURE_2D, plugin->priv->texture_id); + // initialize render texture + // todo: let gst create the texture + const GstGLFuncs *glFunctions = glav->context->gl_vtable; + + glFunctions->GenTextures(1, &plugin->priv->texture_id); + glFunctions->BindTexture(GL_TEXTURE_2D, plugin->priv->texture_id); // allocate texture - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + glFunctions->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE); - glBindTexture(GL_TEXTURE_2D, 0); + glFunctions->BindTexture(GL_TEXTURE_2D, 0); plugin->priv->allocation_params = gst_gl_video_allocation_params_new_wrapped_texture( diff --git a/src/plugin.h b/src/plugin.h index da59250..5683ab0 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -42,7 +42,7 @@ struct _GstProjectM { }; struct _GstProjectMClass { - GstPMAudioVisualizerClass parent_class; + GstGLBaseAudioVisualizerClass parent_class; }; static void gst_projectm_set_property(GObject *object, guint prop_id, From 3c5c8d624a2375e45271a9f2abdae861553bc886 Mon Sep 17 00:00:00 2001 From: hack Date: Sun, 1 Jun 2025 23:21:54 -0500 Subject: [PATCH 22/37] remove duplicate assignment --- src/gstglbaseaudiovisualizer.c | 10 ++++++---- src/gstglbaseaudiovisualizer.h | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 805a017..bcd9964 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -392,19 +392,21 @@ gst_gl_base_audio_visualizer_fill(GstPMAudioVisualizer *bscope, glav->priv->n_frames == 1)) goto eos; + GstBuffer *buffer = video->buffer; + + // the following vars are params for passing values to _fill_gl() // video is mapped to gl memory glav->priv->out_tex = (GstGLMemory *)video->map[0].memory; glav->priv->in_audio = audio; - GstBuffer *buffer = video->buffer; - + // make current presentation timestamp accessible before rendering glav->pts = GST_BUFFER_PTS(buffer); - glav->priv->in_audio = audio; - // dispatch _fill_gl to the gl thread + // dispatch _fill_gl to the gl thread, blocking call gst_gl_context_thread_add(glav->context, (GstGLContextThreadFunc)_fill_gl, glav); + // clear param refs, these pointers never owned the data glav->priv->out_tex = NULL; glav->priv->in_audio = NULL; diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index 8d1e110..39df5c3 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -104,9 +104,9 @@ struct _GstGLBaseAudioVisualizerClass { /*< public >*/ GstGLAPI supported_gl_api; - /* called when gl context starts */ + /* called once the gl context can be used for initializing gl resources */ gboolean (*gl_start)(GstGLBaseAudioVisualizer *glav); - /* called when gl context stops */ + /* called when gl context is being closed and gl resources need to be cleaned up */ void (*gl_stop)(GstGLBaseAudioVisualizer *glav); /* called when caps have been set for the pipeline */ gboolean (*setup)(GstGLBaseAudioVisualizer *glav); From 1bbe36a1e363c7f98e53ba52105a64e8ff02bb1f Mon Sep 17 00:00:00 2001 From: hack Date: Mon, 2 Jun 2025 02:59:09 -0500 Subject: [PATCH 23/37] clang format --- src/gstglbaseaudiovisualizer.c | 8 ++++---- src/gstglbaseaudiovisualizer.h | 5 +++-- src/plugin.c | 12 +++++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index bcd9964..468f05b 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -124,13 +124,13 @@ static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( static gboolean gst_gl_base_audio_visualizer_default_setup(GstGLBaseAudioVisualizer *glav); -/* gl context is started, default v-impl for this class. can be overwritten - * by implementer. */ +/* gl context is started and usable. called from gl thread. default v-impl for + * this class, can be overwritten by implementer. */ static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav); -/* gl context is shutting down, default v-impl for this class. can be - * overwritten by implementer. */ +/* gl context is shutting down. called from gl thread. default v-impl for this + * class. can be overwritten by implementer. */ static void gst_gl_base_audio_visualizer_default_gl_stop(GstGLBaseAudioVisualizer *glav); diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index 39df5c3..1ecf611 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -104,9 +104,10 @@ struct _GstGLBaseAudioVisualizerClass { /*< public >*/ GstGLAPI supported_gl_api; - /* called once the gl context can be used for initializing gl resources */ + /* called from gl thread once the gl context can be used for initializing gl + * resources */ gboolean (*gl_start)(GstGLBaseAudioVisualizer *glav); - /* called when gl context is being closed and gl resources need to be cleaned up */ + /* called from gl thread when gl context is being closed for gl resource up */ void (*gl_stop)(GstGLBaseAudioVisualizer *glav); /* called when caps have been set for the pipeline */ gboolean (*setup)(GstGLBaseAudioVisualizer *glav); diff --git a/src/plugin.c b/src/plugin.c index 9009818..a55ff7e 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -315,14 +315,16 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glFunctions->BindTexture(GL_TEXTURE_2D, plugin->priv->texture_id); // allocate texture - glFunctions->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), - GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, - GL_UNSIGNED_BYTE, NULL); + glFunctions->TexImage2D( + GL_TEXTURE_2D, 0, GL_RGBA, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), + GST_VIDEO_INFO_HEIGHT(&gstav->vinfo), 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL); glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, + GL_CLAMP_TO_EDGE); + glFunctions->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, + GL_CLAMP_TO_EDGE); // glTexEnvi(GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_REPLACE); glFunctions->BindTexture(GL_TEXTURE_2D, 0); From fb5b70815c15ce7e1ac70c4e67e50659f6158a75 Mon Sep 17 00:00:00 2001 From: hack Date: Mon, 2 Jun 2025 18:58:11 -0500 Subject: [PATCH 24/37] docs fix origin ref --- src/gstpmaudiovisualizer.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index cc9caff..2b42987 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -36,14 +36,15 @@ /* * The code in this file is based on * GStreamer / gst-plugins-base / 1.19.2, latest version as of 2025/05/29. - * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: - * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h + * gst-libs/gst/pbutils/gstaudiovisualizer.c Git Repository: + * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.c * Original copyright notice has been retained at the top of this file. * * The code has been modified to improve compatibility with projectM and OpenGL. * * - New apis for implementer-provided memory allocation and video frame - * buffer mapping. Useful for mapping video frames directly to gl memory. + * buffer mapping. Used by gl plugins for mapping video frames directly to gl + * memory. * * - Expose the stream time (dts) state. * From 48ded9b55e2deb84e5c546369498b677ffcb5207 Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 3 Jun 2025 22:40:06 -0500 Subject: [PATCH 25/37] docs fix --- src/gstglbaseaudiovisualizer.c | 29 ++++++++++++++++++----------- src/gstpmaudiovisualizer.c | 8 +++++++- src/plugin.h | 6 ++++-- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 468f05b..3293467 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -50,9 +50,16 @@ * #GstGLBaseAudioVisualizer handles the nitty gritty details of retrieving an * OpenGL context. It also provides `gl_start()` and `gl_stop()` virtual methods * that ensure an OpenGL context is available and current in the calling thread - * for initializing and cleaning up OpenGL dependent resources. The `render` + * for initializing and cleaning up OpenGL resources. The `render` * virtual method of the GstPMAudioVisualizer is implemented to perform OpenGL - * rendering. fill_gl_memory is called to render directly to gl memory. + * rendering. The implementer provides an implementation for fill_gl_memory to + * render directly to gl memory. + * + * Typical plug-in call order for implementer-provided functions: + * - setup (once) + * - gl_start (once) + * - fill_gl_memory (once for each frame) + * - gl_stop (once) */ #define GST_CAT_DEFAULT gst_gl_base_audio_visualizer_debug @@ -119,23 +126,23 @@ static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav); static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( GstPMAudioVisualizer *gstav, GstQuery *query); -/* called when format changes, default v-impl for this class. can be overwritten - * by implementer. */ +/* called when format changes, default empty v-impl for this class. can be + * overwritten by implementer. */ static gboolean gst_gl_base_audio_visualizer_default_setup(GstGLBaseAudioVisualizer *glav); -/* gl context is started and usable. called from gl thread. default v-impl for - * this class, can be overwritten by implementer. */ +/* gl context is started and usable. called from gl thread. default empty v-impl + * for this class, can be overwritten by implementer. */ static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav); -/* gl context is shutting down. called from gl thread. default v-impl for this - * class. can be overwritten by implementer. */ +/* gl context is shutting down. called from gl thread. default empty v-impl for + * this class. can be overwritten by implementer. */ static void gst_gl_base_audio_visualizer_default_gl_stop(GstGLBaseAudioVisualizer *glav); -/* default empty v-impl for rendering a frame. can be overwritten by - * implementer. */ +/* default empty v-impl for rendering a frame. called from gl thread. can be + * overwritten by implementer. */ static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( GstGLBaseAudioVisualizer *glav, GstBuffer *in_audio, GstGLMemory *mem); @@ -669,7 +676,7 @@ static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( } if (!pool || !GST_IS_GL_BUFFER_POOL(pool)) { - // can't use this pool + /* can't use this pool */ if (pool) gst_object_unref(pool); pool = gst_gl_buffer_pool_new(context); diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 2b42987..530d745 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -54,8 +54,14 @@ * * - Bugfix for the amount of bytes being flushed for a single video frame from * the audio input buffer. + * + * Typical plug-in call order for implementer-provided functions: + * - decide_allocation (once) + * - setup (once) + * - prepare_output_buffer (once for each frame) + * - map_output_buffer (once for each frame) + * - render (once for each frame) */ - #ifdef HAVE_CONFIG_H #include "config.h" #endif diff --git a/src/plugin.h b/src/plugin.h index 5683ab0..827cbe9 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -13,8 +13,10 @@ G_DECLARE_FINAL_TYPE(GstProjectM, gst_projectm, GST, PROJECTM, GstGLBaseAudioVisualizer) /* - * Main plug-in. Handles interactions with projectM. Extends plug-ins for gl - * context handling and audio-visualization (timing, video frame data). + * Main plug-in. Handles interactions with projectM. + * Uses GstPMAudioVisualizer for handling audio-visualization (audio input, + * timing, video frame data). GstGLBaseAudioVisualizer extends + * GstPMAudioVisualizer to add gl context handling and is used by this plugin directly. * GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. */ struct _GstProjectM { From 39346ebb7b80b3d3d55ae914720394e0ee3119ad Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 3 Jun 2025 23:00:25 -0500 Subject: [PATCH 26/37] reduce qos frame skip time to just continue --- src/gstpmaudiovisualizer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 530d745..e33affd 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -828,8 +828,10 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, if (diff >= 0) /* we're late, this is a good estimate for next displayable * frame (see part-qos.txt) */ + // original calc seems like a lot: timestamp + diff * 2 + scope->priv->frame_duration; + // let's just continue where we are now scope->priv->earliest_time = - timestamp + 2 * diff + scope->priv->frame_duration; + timestamp + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; GST_OBJECT_UNLOCK(scope); From 95f4d85e5e01917bbe2716d773caa5369d030c76 Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 3 Jun 2025 23:04:45 -0500 Subject: [PATCH 27/37] reduce qos frame skip time docs --- src/gstpmaudiovisualizer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index e33affd..b554316 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -55,6 +55,8 @@ * - Bugfix for the amount of bytes being flushed for a single video frame from * the audio input buffer. * + * - Bugfix for accumulating qos frame drops while real-time rendering. + * * Typical plug-in call order for implementer-provided functions: * - decide_allocation (once) * - setup (once) From 3280bec4482e4908b8448b577a4e12c95cd83ff0 Mon Sep 17 00:00:00 2001 From: hack Date: Tue, 3 Jun 2025 23:46:21 -0500 Subject: [PATCH 28/37] clang format --- src/gstpmaudiovisualizer.c | 8 +++++--- src/plugin.h | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index b554316..f7eddbe 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -157,6 +157,8 @@ struct _GstPMAudioVisualizerPrivate { /* QoS stuff */ /* with LOCK */ gdouble proportion; + /* qos: earliest time to render the next frame, the render loop will skip + * frames until this time */ GstClockTime earliest_time; guint dropped; /* frames dropped / not dropped */ @@ -830,10 +832,10 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, if (diff >= 0) /* we're late, this is a good estimate for next displayable * frame (see part-qos.txt) */ - // original calc seems like a lot: timestamp + diff * 2 + scope->priv->frame_duration; + // bugfix, original calc seems like a lot: + // timestamp + diff * 2 + scope->priv->frame_duration; // let's just continue where we are now - scope->priv->earliest_time = - timestamp + scope->priv->frame_duration; + scope->priv->earliest_time = timestamp + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; GST_OBJECT_UNLOCK(scope); diff --git a/src/plugin.h b/src/plugin.h index 827cbe9..b46977c 100644 --- a/src/plugin.h +++ b/src/plugin.h @@ -16,8 +16,8 @@ G_DECLARE_FINAL_TYPE(GstProjectM, gst_projectm, GST, PROJECTM, * Main plug-in. Handles interactions with projectM. * Uses GstPMAudioVisualizer for handling audio-visualization (audio input, * timing, video frame data). GstGLBaseAudioVisualizer extends - * GstPMAudioVisualizer to add gl context handling and is used by this plugin directly. - * GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. + * GstPMAudioVisualizer to add gl context handling and is used by this plugin + * directly. GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. */ struct _GstProjectM { GstGLBaseAudioVisualizer element; From 720bc9aca73f8a50304431e9a8aa5c45d6535a12 Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 5 Jun 2025 01:18:15 -0500 Subject: [PATCH 29/37] backport bugfixes from gst and additional fixes --- src/gstpmaudiovisualizer.c | 35 +++++++++++++++++------------------ src/gstpmaudiovisualizer.h | 4 ++-- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index f7eddbe..3503052 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -35,9 +35,9 @@ /* * The code in this file is based on - * GStreamer / gst-plugins-base / 1.19.2, latest version as of 2025/05/29. + * GStreamer / gst-plugins-base, latest version as of 2025/05/29. * gst-libs/gst/pbutils/gstaudiovisualizer.c Git Repository: - * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.c + * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/blob/main/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c * Original copyright notice has been retained at the top of this file. * * The code has been modified to improve compatibility with projectM and OpenGL. @@ -55,7 +55,7 @@ * - Bugfix for the amount of bytes being flushed for a single video frame from * the audio input buffer. * - * - Bugfix for accumulating qos frame drops while real-time rendering. + * - Bugfix for long qos frame drops while real-time rendering. * * Typical plug-in call order for implementer-provided functions: * - decide_allocation (once) @@ -619,10 +619,11 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GstFlowReturn ret = GST_FLOW_OK; GstPMAudioVisualizer *scope; GstPMAudioVisualizerClass *klass; - GstBuffer *inbuf; guint64 dist, ts; guint avail, sbpf; - gpointer adata; + // databuf is a buffer pointing to single video frame worth of audio data from the adapter + // inbuf is a buffer holding a copy of the current single video frame worth of audio data from the adapter to process + GstBuffer *databuf, *inbuf; gint bpf, rate; scope = GST_PM_AUDIO_VISUALIZER(parent); @@ -682,13 +683,10 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, if (GST_CLOCK_TIME_IS_VALID(ts)) { GstClockTime earliest_time; gdouble proportion; - guint64 qostime; - guint64 running_time; - - running_time = gst_segment_to_running_time(&scope->priv->segment, - GST_FORMAT_TIME, ts); + gint64 qostime; - qostime = running_time + scope->priv->frame_duration; + qostime = gst_segment_to_running_time(&scope->priv->segment, + GST_FORMAT_TIME, ts) + scope->priv->frame_duration; GST_OBJECT_LOCK(scope); earliest_time = scope->priv->earliest_time; @@ -744,16 +742,16 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GST_BUFFER_DURATION(outbuf) = scope->priv->frame_duration; /* this can fail as the data size we need could have changed */ - if (!(adata = (gpointer)gst_adapter_map(scope->priv->adapter, sbpf))) + if (!(databuf = gst_adapter_get_buffer(scope->priv->adapter, sbpf))) break; /* allow customized memory to video frame mapping */ klass->map_output_buffer(scope, &outframe, outbuf); /* place sbpf number of bytes of audio data into inbuf */ - gst_buffer_replace_all_memory( - inbuf, gst_memory_new_wrapped(GST_MEMORY_FLAG_READONLY, adata, sbpf, 0, - sbpf, NULL, NULL)); + gst_buffer_remove_all_memory (inbuf); + gst_buffer_copy_into (inbuf, databuf, GST_BUFFER_COPY_MEMORY, 0, sbpf); + gst_buffer_unref (databuf); /* call class->render() vmethod */ if (klass->render) { @@ -777,8 +775,8 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, /* we want to take less or more, depending on spf : req_spf */ if (avail - sbpf >= sbpf) { // enough audio data for more frames is available - gst_adapter_flush(scope->priv->adapter, sbpf); gst_adapter_unmap(scope->priv->adapter); + gst_adapter_flush(scope->priv->adapter, sbpf); } else if (avail >= sbpf) { // was just enough audio data for one frame /* just flush a bit and stop */ @@ -788,8 +786,8 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, // instead just flush one video frame worth of audio data from the buffer // and stop - gst_adapter_flush(scope->priv->adapter, sbpf); gst_adapter_unmap(scope->priv->adapter); + gst_adapter_flush(scope->priv->adapter, sbpf); break; } avail = gst_adapter_available(scope->priv->adapter); @@ -834,7 +832,8 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, * frame (see part-qos.txt) */ // bugfix, original calc seems like a lot: // timestamp + diff * 2 + scope->priv->frame_duration; - // let's just continue where we are now + // a bugfix has been added since to limit drops to second: scope->priv->earliest_time = timestamp + MIN (2 * diff, GST_SECOND) + scope->priv->frame_duration; + // let's just continue with the next frame from where we are now scope->priv->earliest_time = timestamp + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index 36fcaa8..d1f8960 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -22,9 +22,9 @@ /* * The code in this file is based on - * GStreamer / gst-plugins-base / 1.19.2, latest version as of 2025/05/29. + * GStreamer / gst-plugins-base, latest version as of 2025/05/29. * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: - * https://github.com/GStreamer/gst-plugins-base/blob/master/gst-libs/gst/pbutils/gstaudiovisualizer.h + * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/blob/main/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.h * * Original copyright notice has been retained at the top of this file. * The code has been modified to improve compatibility with projectM and OpenGL. From a41f2186b93a6deba4f8cc81f18761a699e181fa Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 5 Jun 2025 01:18:34 -0500 Subject: [PATCH 30/37] backport bugfixes from gst --- src/gstglbaseaudiovisualizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 3293467..4befdc8 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -365,7 +365,7 @@ static void gst_gl_base_audio_visualizer_parent_map_output_buffer( GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf) { /* map video to gl memory */ gst_video_frame_map(outframe, &scope->vinfo, outbuf, - GST_MAP_WRITE | GST_MAP_GL); + GST_MAP_WRITE | GST_MAP_GL | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); } static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( From aa676279d003e137090855f3e2c335776977bc43 Mon Sep 17 00:00:00 2001 From: hack Date: Thu, 5 Jun 2025 01:20:04 -0500 Subject: [PATCH 31/37] fix log --- src/projectm.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/projectm.c b/src/projectm.c index 149a5b6..8939bb5 100644 --- a/src/projectm.c +++ b/src/projectm.c @@ -43,7 +43,8 @@ projectm_handle projectm_init(GstProjectM *plugin) { // &ProjectMWrapper::PresetSwitchedEvent, static_cast(this)); } else { GST_DEBUG_OBJECT(plugin, "Playlist disabled"); - } // Log properties + } + // Log properties GST_INFO_OBJECT( plugin, "Using Properties: " @@ -61,12 +62,13 @@ projectm_handle projectm_init(GstProjectM *plugin) { "preset-locked=%d, " "enable-playlist=%d, " "shuffle-presets=%d", + "pts-sync=%d", plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, plugin->hard_cut_duration, plugin->hard_cut_enabled, plugin->hard_cut_sensitivity, plugin->soft_cut_duration, plugin->preset_duration, plugin->mesh_width, plugin->mesh_height, plugin->aspect_correction, plugin->easter_egg, plugin->preset_locked, - plugin->enable_playlist, plugin->shuffle_presets); + plugin->enable_playlist, plugin->shuffle_presets, plugin->pts_sync); // Load preset file if path is provided if (plugin->preset_path != NULL) { From 80e10784551390946ce2e213b890dbbc651f11fe Mon Sep 17 00:00:00 2001 From: hack Date: Fri, 6 Jun 2025 21:22:51 -0500 Subject: [PATCH 32/37] clang format --- src/gstglbaseaudiovisualizer.c | 3 +- src/gstpmaudiovisualizer.c | 56 ++++++++++++++++++++++------------ src/gstpmaudiovisualizer.h | 2 -- src/projectm.c | 48 ++++++++++++++--------------- 4 files changed, 62 insertions(+), 47 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 4befdc8..e73c20d 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -365,7 +365,8 @@ static void gst_gl_base_audio_visualizer_parent_map_output_buffer( GstPMAudioVisualizer *scope, GstVideoFrame *outframe, GstBuffer *outbuf) { /* map video to gl memory */ gst_video_frame_map(outframe, &scope->vinfo, outbuf, - GST_MAP_WRITE | GST_MAP_GL | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); + GST_MAP_WRITE | GST_MAP_GL | + GST_VIDEO_FRAME_MAP_FLAG_NO_REF); } static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 3503052..a8ddddc 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -78,9 +78,6 @@ GST_DEBUG_CATEGORY_STATIC(pm_audio_visualizer_debug); #define GST_CAT_DEFAULT (pm_audio_visualizer_debug) -#define DEFAULT_SHADER GST_AUDIO_VISUALIZER_SHADER_FADE -#define DEFAULT_SHADE_AMOUNT 0x000a0a0a - enum { PROP_0 }; static GstBaseTransformClass *parent_class = NULL; @@ -203,7 +200,7 @@ GType gst_pm_audio_visualizer_get_type(void) { } static inline GstPMAudioVisualizerPrivate * -gst_audio_visualizer_get_instance_private(GstPMAudioVisualizer *self) { +gst_pm_audio_visualizer_get_instance_private(GstPMAudioVisualizer *self) { return (G_STRUCT_MEMBER_P(self, private_offset)); } @@ -239,7 +236,7 @@ static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, GstPMAudioVisualizerClass *g_class) { GstPadTemplate *pad_template; - scope->priv = gst_audio_visualizer_get_instance_private(scope); + scope->priv = gst_pm_audio_visualizer_get_instance_private(scope); /* create the sink and src pads */ pad_template = @@ -324,7 +321,7 @@ static void gst_pm_audio_visualizer_dispose(GObject *object) { G_OBJECT_CLASS(parent_class)->dispose(object); } -static void gst_audio_visualizer_reset(GstPMAudioVisualizer *scope) { +static void gst_pm_audio_visualizer_reset(GstPMAudioVisualizer *scope) { gst_adapter_clear(scope->priv->adapter); gst_segment_init(&scope->priv->segment, GST_FORMAT_UNDEFINED); @@ -469,8 +466,22 @@ no_format: { } } +void gst_pm_audio_visualizer_on_pad_added(GstElement *src, GstPad *new_pad, + gpointer data) { + GstElement *sink = GST_ELEMENT(data); + GstPad *sink_pad = gst_element_get_static_pad(sink, "sink"); + + if (!gst_pad_is_linked(sink_pad)) { + if (gst_pad_link(new_pad, sink_pad) != GST_PAD_LINK_OK) { + g_warning("Failed to link pads"); + } + } + + gst_object_unref(sink_pad); +} + /* takes ownership of the pool, allocator and query */ -static gboolean gst_audio_visualizer_set_allocation( +static gboolean gst_pm_audio_visualizer_set_allocation( GstPMAudioVisualizer *scope, GstBufferPool *pool, GstAllocator *allocator, const GstAllocationParams *params, GstQuery *query) { GstAllocator *oldalloc; @@ -554,8 +565,8 @@ gst_pm_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); /* now store */ - result = gst_audio_visualizer_set_allocation(scope, pool, allocator, ¶ms, - query); + result = gst_pm_audio_visualizer_set_allocation(scope, pool, allocator, + ¶ms, query); return result; @@ -621,8 +632,10 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GstPMAudioVisualizerClass *klass; guint64 dist, ts; guint avail, sbpf; - // databuf is a buffer pointing to single video frame worth of audio data from the adapter - // inbuf is a buffer holding a copy of the current single video frame worth of audio data from the adapter to process + // databuf is a buffer holding to one video frame worth of audio data used as + // temp buffer for copying from the adapter only + // inbuf is a plugin-scoped buffer holding a copy of the one video frame worth + // of audio data from the adapter to process GstBuffer *databuf, *inbuf; gint bpf, rate; @@ -686,7 +699,8 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, gint64 qostime; qostime = gst_segment_to_running_time(&scope->priv->segment, - GST_FORMAT_TIME, ts) + scope->priv->frame_duration; + GST_FORMAT_TIME, ts) + + scope->priv->frame_duration; GST_OBJECT_LOCK(scope); earliest_time = scope->priv->earliest_time; @@ -749,9 +763,9 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, klass->map_output_buffer(scope, &outframe, outbuf); /* place sbpf number of bytes of audio data into inbuf */ - gst_buffer_remove_all_memory (inbuf); - gst_buffer_copy_into (inbuf, databuf, GST_BUFFER_COPY_MEMORY, 0, sbpf); - gst_buffer_unref (databuf); + gst_buffer_remove_all_memory(inbuf); + gst_buffer_copy_into(inbuf, databuf, GST_BUFFER_COPY_MEMORY, 0, sbpf); + gst_buffer_unref(databuf); /* call class->render() vmethod */ if (klass->render) { @@ -832,8 +846,10 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, * frame (see part-qos.txt) */ // bugfix, original calc seems like a lot: // timestamp + diff * 2 + scope->priv->frame_duration; - // a bugfix has been added since to limit drops to second: scope->priv->earliest_time = timestamp + MIN (2 * diff, GST_SECOND) + scope->priv->frame_duration; - // let's just continue with the next frame from where we are now + // a bugfix has been added since to limit drops to second: + // scope->priv->earliest_time = timestamp + MIN (2 * diff, GST_SECOND) + + // scope->priv->frame_duration; let's just continue with the next frame + // from where we are now scope->priv->earliest_time = timestamp + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; @@ -873,7 +889,7 @@ static gboolean gst_pm_audio_visualizer_sink_event(GstPad *pad, break; } case GST_EVENT_FLUSH_STOP: - gst_audio_visualizer_reset(scope); + gst_pm_audio_visualizer_reset(scope); res = gst_pad_push_event(scope->priv->srcpad, event); break; case GST_EVENT_SEGMENT: { @@ -961,7 +977,7 @@ gst_pm_audio_visualizer_change_state(GstElement *element, switch (transition) { case GST_STATE_CHANGE_READY_TO_PAUSED: - gst_audio_visualizer_reset(scope); + gst_pm_audio_visualizer_reset(scope); break; default: break; @@ -971,7 +987,7 @@ gst_pm_audio_visualizer_change_state(GstElement *element, switch (transition) { case GST_STATE_CHANGE_PAUSED_TO_READY: - gst_audio_visualizer_set_allocation(scope, NULL, NULL, NULL, NULL); + gst_pm_audio_visualizer_set_allocation(scope, NULL, NULL, NULL, NULL); break; case GST_STATE_CHANGE_READY_TO_NULL: break; diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index d1f8960..e88e11f 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -74,8 +74,6 @@ struct _GstPMAudioVisualizer { guint64 stream_time; /*< private >*/ - gpointer _padding[GST_PADDING]; - GstPMAudioVisualizerPrivate *priv; }; diff --git a/src/projectm.c b/src/projectm.c index 8939bb5..78fe2b9 100644 --- a/src/projectm.c +++ b/src/projectm.c @@ -45,30 +45,30 @@ projectm_handle projectm_init(GstProjectM *plugin) { GST_DEBUG_OBJECT(plugin, "Playlist disabled"); } // Log properties - GST_INFO_OBJECT( - plugin, - "Using Properties: " - "preset=%s, " - "texture-dir=%s, " - "beat-sensitivity=%f, " - "hard-cut-duration=%f, " - "hard-cut-enabled=%d, " - "hard-cut-sensitivity=%f, " - "soft-cut-duration=%f, " - "preset-duration=%f, " - "mesh-size=(%lu, %lu)" - "aspect-correction=%d, " - "easter-egg=%f, " - "preset-locked=%d, " - "enable-playlist=%d, " - "shuffle-presets=%d", - "pts-sync=%d", - plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, - plugin->hard_cut_duration, plugin->hard_cut_enabled, - plugin->hard_cut_sensitivity, plugin->soft_cut_duration, - plugin->preset_duration, plugin->mesh_width, plugin->mesh_height, - plugin->aspect_correction, plugin->easter_egg, plugin->preset_locked, - plugin->enable_playlist, plugin->shuffle_presets, plugin->pts_sync); + GST_INFO_OBJECT(plugin, + "Using Properties: " + "preset=%s, " + "texture-dir=%s, " + "beat-sensitivity=%f, " + "hard-cut-duration=%f, " + "hard-cut-enabled=%d, " + "hard-cut-sensitivity=%f, " + "soft-cut-duration=%f, " + "preset-duration=%f, " + "mesh-size=(%lu, %lu)" + "aspect-correction=%d, " + "easter-egg=%f, " + "preset-locked=%d, " + "enable-playlist=%d, " + "shuffle-presets=%d", + "pts-sync=%d", plugin->preset_path, plugin->texture_dir_path, + plugin->beat_sensitivity, plugin->hard_cut_duration, + plugin->hard_cut_enabled, plugin->hard_cut_sensitivity, + plugin->soft_cut_duration, plugin->preset_duration, + plugin->mesh_width, plugin->mesh_height, + plugin->aspect_correction, plugin->easter_egg, + plugin->preset_locked, plugin->enable_playlist, + plugin->shuffle_presets, plugin->pts_sync); // Load preset file if path is provided if (plugin->preset_path != NULL) { From d51c9801666ebf71d4602dd2f8697821a77c28d0 Mon Sep 17 00:00:00 2001 From: hack Date: Sat, 14 Jun 2025 16:44:22 -0500 Subject: [PATCH 33/37] install timestamp offset property --- src/gstglbaseaudiovisualizer.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index e73c20d..04dc7ea 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -65,6 +65,8 @@ #define GST_CAT_DEFAULT gst_gl_base_audio_visualizer_debug GST_DEBUG_CATEGORY_STATIC(GST_CAT_DEFAULT); +#define DEFAULT_TIMESTAMP_OFFSET 0 + struct _GstGLBaseAudioVisualizerPrivate { GstGLContext *other_context; GstGLMemory *out_tex; @@ -215,6 +217,14 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { klass->prepare_output_buffer = GST_DEBUG_FUNCPTR( gst_gl_base_audio_visualizer_default_prepare_output_buffer); + + g_object_class_install_property( + gobject_class, PROP_TIMESTAMP_OFFSET, + g_param_spec_int64( + "timestamp-offset", "Timestamp Offset", + "Specifies initial offset for the stream timestamp.", 0, G_MAXINT64, + DEFAULT_TIMESTAMP_OFFSET, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + } static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { @@ -225,7 +235,6 @@ static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv->out_tex = NULL; glav->context = NULL; glav->pts = 0; - glav->priv->timestamp_offset = 0; g_rec_mutex_init(&glav->priv->context_lock); gst_gl_base_audio_visualizer_start(glav); } From e48a6961b3ce782b5d936b2c18fa66bfad93a554 Mon Sep 17 00:00:00 2001 From: hack Date: Mon, 16 Jun 2025 00:03:46 -0500 Subject: [PATCH 34/37] add projectm plugin locking clenup playlist handle --- src/plugin.c | 20 +++++++++++++++++--- src/projectm.c | 11 +++++++---- src/projectm.h | 4 +++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/plugin.c b/src/plugin.c index a55ff7e..fe01210 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -21,9 +21,13 @@ GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); #define GST_CAT_DEFAULT gst_projectm_debug +#define GST_PROJECTM_LOCK(plugin) (g_mutex_lock(&plugin->priv->projectm_lock)) +#define GST_PROJECTM_UNLOCK(plugin) (g_mutex_unlock(&plugin->priv->projectm_lock)) struct _GstProjectMPrivate { projectm_handle handle; + projectm_playlist_handle playlist; + GMutex projectm_lock; GstClockTime first_frame_time; gboolean first_frame_received; @@ -79,9 +83,10 @@ static GstFlowReturn gst_projectm_prepare_output_buffer(GstGLBaseAudioVisualizer *scope, GstBuffer **outbuf) { GstProjectM *plugin = GST_PROJECTM(scope); - + GST_PROJECTM_LOCK(plugin); *outbuf = wrap_gl_texture(scope, plugin); GST_DEBUG_OBJECT(plugin, "Wrapped RT texture buffer"); + GST_PROJECTM_UNLOCK(plugin); return GST_FLOW_OK; } @@ -261,17 +266,20 @@ static void gst_projectm_init(GstProjectM *plugin) { plugin->priv->in_audio = NULL; plugin->priv->mem = NULL; plugin->priv->allocation_params = NULL; + g_mutex_init(&plugin->priv->projectm_lock); } static void gst_projectm_finalize(GObject *object) { GstProjectM *plugin = GST_PROJECTM(object); g_free(plugin->preset_path); g_free(plugin->texture_dir_path); + g_mutex_clear(&plugin->priv->projectm_lock); G_OBJECT_CLASS(gst_projectm_parent_class)->finalize(object); } static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { GstProjectM *plugin = GST_PROJECTM(src); + GST_PROJECTM_LOCK(plugin); if (plugin->priv->handle) { GST_DEBUG_OBJECT(plugin, "Destroying ProjectM instance"); projectm_destroy(plugin->priv->handle); @@ -291,6 +299,7 @@ static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { gst_gl_video_allocation_params_free_data(plugin->priv->allocation_params); plugin->priv->allocation_params = NULL; } + GST_PROJECTM_UNLOCK(plugin); } static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { @@ -307,6 +316,7 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { } #endif + GST_PROJECTM_LOCK(plugin); // initialize render texture // todo: let gst create the texture const GstGLFuncs *glFunctions = glav->context->gl_vtable; @@ -336,9 +346,8 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { // Check if ProjectM instance exists, and create if not if (!plugin->priv->handle) { // Create ProjectM instance - plugin->priv->handle = projectm_init(plugin); plugin->priv->first_frame_received = FALSE; - if (!plugin->priv->handle) { + if (!projectm_init(plugin, &plugin->priv->handle, &plugin->priv->playlist)) { GST_ERROR_OBJECT(plugin, "ProjectM could not be initialized"); return FALSE; } @@ -349,6 +358,7 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { glav->context, GST_VIDEO_INFO_WIDTH(&gstav->vinfo), GST_VIDEO_INFO_HEIGHT(&gstav->vinfo)); + GST_PROJECTM_UNLOCK(plugin); GST_INFO_OBJECT(plugin, "GL start complete"); return TRUE; } @@ -455,6 +465,8 @@ static gboolean gst_projectm_fill_gl_memory(GstGLBaseAudioVisualizer *glav, GstProjectM *plugin = GST_PROJECTM(glav); + GST_PROJECTM_LOCK(plugin); + plugin->priv->in_audio = in_audio; plugin->priv->mem = mem; @@ -464,6 +476,8 @@ static gboolean gst_projectm_fill_gl_memory(GstGLBaseAudioVisualizer *glav, plugin->priv->in_audio = NULL; plugin->priv->mem = NULL; + GST_PROJECTM_UNLOCK(plugin); + return result; } diff --git a/src/projectm.c b/src/projectm.c index 78fe2b9..16d89c3 100644 --- a/src/projectm.c +++ b/src/projectm.c @@ -13,7 +13,8 @@ GST_DEBUG_CATEGORY_STATIC(projectm_debug); #define GST_CAT_DEFAULT projectm_debug -projectm_handle projectm_init(GstProjectM *plugin) { +bool projectm_init(GstProjectM *plugin, projectm_handle *ret_handle, + projectm_playlist_handle *ret_playlist) { projectm_handle handle = NULL; projectm_playlist_handle playlist = NULL; GST_DEBUG_CATEGORY_INIT(projectm_debug, "projectm", 0, "ProjectM"); @@ -28,16 +29,18 @@ projectm_handle projectm_init(GstProjectM *plugin) { GST_DEBUG_OBJECT( plugin, "project_create() returned NULL, projectM instance was not created!"); - return NULL; + return FALSE; } else { GST_DEBUG_OBJECT(plugin, "Created projectM instance!"); } + *ret_handle = handle; if (plugin->enable_playlist) { GST_DEBUG_OBJECT(plugin, "Playlist enabled"); // initialize preset playlist playlist = projectm_playlist_create(handle); + *ret_playlist = playlist; projectm_playlist_set_shuffle(playlist, plugin->shuffle_presets); // projectm_playlist_set_preset_switched_event_callback(_playlist, // &ProjectMWrapper::PresetSwitchedEvent, static_cast(this)); @@ -72,7 +75,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { // Load preset file if path is provided if (plugin->preset_path != NULL) { - int added_count = + unsigned int added_count = projectm_playlist_add_path(playlist, plugin->preset_path, true, false); GST_INFO("Loaded preset path: %s, presets found: %d", plugin->preset_path, added_count); @@ -115,7 +118,7 @@ projectm_handle projectm_init(GstProjectM *plugin) { projectm_set_window_size(handle, GST_VIDEO_INFO_WIDTH(&bscope->vinfo), GST_VIDEO_INFO_HEIGHT(&bscope->vinfo)); - return handle; + return TRUE; } // void projectm_render(GstProjectM *plugin, gint16 *samples, gint sample_count) diff --git a/src/projectm.h b/src/projectm.h index 1ba6a37..79efc32 100644 --- a/src/projectm.h +++ b/src/projectm.h @@ -5,13 +5,15 @@ #include "plugin.h" #include +#include G_BEGIN_DECLS /** * @brief Initialize ProjectM */ -projectm_handle projectm_init(GstProjectM *plugin); +bool projectm_init(GstProjectM *plugin, projectm_handle *handle, + projectm_playlist_handle *playlist); /** * @brief Render ProjectM From 88e6c575d18af88451cb866e18be03bd604a03ca Mon Sep 17 00:00:00 2001 From: hack Date: Mon, 16 Jun 2025 00:03:58 -0500 Subject: [PATCH 35/37] additional locking --- src/gstpmaudiovisualizer.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index a8ddddc..0a48ee7 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -341,7 +341,9 @@ gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, if (!gst_audio_info_from_caps(&info, caps)) goto wrong_caps; + g_mutex_lock(&scope->priv->config_lock); scope->ainfo = info; + g_mutex_unlock(&scope->priv->config_lock); GST_DEBUG_OBJECT(scope, "audio: channels %d, rate %d", GST_AUDIO_INFO_CHANNELS(&info), GST_AUDIO_INFO_RATE(&info)); @@ -374,6 +376,8 @@ static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, klass = GST_PM_AUDIO_VISUALIZER_CLASS(G_OBJECT_GET_CLASS(scope)); + g_mutex_lock(&scope->priv->config_lock); + scope->vinfo = info; scope->priv->frame_duration = gst_util_uint64_scale_int( @@ -383,6 +387,8 @@ static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, GST_VIDEO_INFO_FPS_N(&info)); scope->req_spf = scope->priv->spf; + g_mutex_unlock(&scope->priv->config_lock); + if (klass->setup && !klass->setup(scope)) goto setup_failed; @@ -760,7 +766,9 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, break; /* allow customized memory to video frame mapping */ + g_mutex_unlock(&scope->priv->config_lock); klass->map_output_buffer(scope, &outframe, outbuf); + g_mutex_lock(&scope->priv->config_lock); /* place sbpf number of bytes of audio data into inbuf */ gst_buffer_remove_all_memory(inbuf); @@ -769,11 +777,14 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, /* call class->render() vmethod */ if (klass->render) { + g_mutex_unlock(&scope->priv->config_lock); if (!klass->render(scope, inbuf, &outframe)) { + g_mutex_lock(&scope->priv->config_lock); ret = GST_FLOW_ERROR; gst_video_frame_unmap(&outframe); goto beach; } + g_mutex_lock(&scope->priv->config_lock); } gst_video_frame_unmap(&outframe); From 36dd8773187d770fd9a54888aa61ba1695ae016a Mon Sep 17 00:00:00 2001 From: hack Date: Mon, 16 Jun 2025 00:49:07 -0500 Subject: [PATCH 36/37] dynamic qos event calc --- src/gstpmaudiovisualizer.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 0a48ee7..50c438a 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -852,16 +852,18 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, /* save stuff for the _chain() function */ GST_OBJECT_LOCK(scope); scope->priv->proportion = proportion; - if (diff >= 0) + if (diff > 0) /* we're late, this is a good estimate for next displayable * frame (see part-qos.txt) */ // bugfix, original calc seems like a lot: // timestamp + diff * 2 + scope->priv->frame_duration; // a bugfix has been added since to limit drops to second: // scope->priv->earliest_time = timestamp + MIN (2 * diff, GST_SECOND) + - // scope->priv->frame_duration; let's just continue with the next frame - // from where we are now - scope->priv->earliest_time = timestamp + scope->priv->frame_duration; + // scope->priv->frame_duration; + // the proposed one second is still way too much for us + // just allow dropping a few frames + scope->priv->earliest_time = timestamp + + MIN(2 * diff, scope->priv->frame_duration * 2) + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; GST_OBJECT_UNLOCK(scope); From a4144a154f889d178e6f937cafecb04cca75bcbc Mon Sep 17 00:00:00 2001 From: hack Date: Mon, 16 Jun 2025 02:06:39 -0500 Subject: [PATCH 37/37] reformat --- src/gstglbaseaudiovisualizer.c | 11 +++++------ src/gstpmaudiovisualizer.c | 5 +++-- src/plugin.c | 8 +++++--- src/projectm.h | 3 --- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 04dc7ea..d48dc1c 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -219,12 +219,11 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { gst_gl_base_audio_visualizer_default_prepare_output_buffer); g_object_class_install_property( - gobject_class, PROP_TIMESTAMP_OFFSET, - g_param_spec_int64( - "timestamp-offset", "Timestamp Offset", - "Specifies initial offset for the stream timestamp.", 0, G_MAXINT64, - DEFAULT_TIMESTAMP_OFFSET, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - + gobject_class, PROP_TIMESTAMP_OFFSET, + g_param_spec_int64("timestamp-offset", "Timestamp Offset", + "Specifies initial offset for the stream timestamp.", + 0, G_MAXINT64, DEFAULT_TIMESTAMP_OFFSET, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 50c438a..f781c9e 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -862,8 +862,9 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, // scope->priv->frame_duration; // the proposed one second is still way too much for us // just allow dropping a few frames - scope->priv->earliest_time = timestamp + - MIN(2 * diff, scope->priv->frame_duration * 2) + scope->priv->frame_duration; + scope->priv->earliest_time = + timestamp + MIN(2 * diff, scope->priv->frame_duration * 2) + + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; GST_OBJECT_UNLOCK(scope); diff --git a/src/plugin.c b/src/plugin.c index fe01210..0cd0500 100644 --- a/src/plugin.c +++ b/src/plugin.c @@ -21,8 +21,9 @@ GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); #define GST_CAT_DEFAULT gst_projectm_debug -#define GST_PROJECTM_LOCK(plugin) (g_mutex_lock(&plugin->priv->projectm_lock)) -#define GST_PROJECTM_UNLOCK(plugin) (g_mutex_unlock(&plugin->priv->projectm_lock)) +#define GST_PROJECTM_LOCK(plugin) (g_mutex_lock(&plugin->priv->projectm_lock)) +#define GST_PROJECTM_UNLOCK(plugin) \ + (g_mutex_unlock(&plugin->priv->projectm_lock)) struct _GstProjectMPrivate { projectm_handle handle; @@ -347,7 +348,8 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { if (!plugin->priv->handle) { // Create ProjectM instance plugin->priv->first_frame_received = FALSE; - if (!projectm_init(plugin, &plugin->priv->handle, &plugin->priv->playlist)) { + if (!projectm_init(plugin, &plugin->priv->handle, + &plugin->priv->playlist)) { GST_ERROR_OBJECT(plugin, "ProjectM could not be initialized"); return FALSE; } diff --git a/src/projectm.h b/src/projectm.h index 79efc32..246b633 100644 --- a/src/projectm.h +++ b/src/projectm.h @@ -1,10 +1,7 @@ #ifndef __PROJECTM_H__ #define __PROJECTM_H__ -#include - #include "plugin.h" -#include #include G_BEGIN_DECLS