From 3c6577cfd36bab21ef321404e0d075ea4228b3e6 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 6 Oct 2025 01:00:20 -0500 Subject: [PATCH 01/32] refactor plugin to use fbo rendering --- CMakeLists.txt | 22 +- README.md | 97 ++- convert.sh | 6 +- example/CMakeLists.txt | 30 + example/dynpads.c | 238 ++++++ src/caps.c | 51 -- src/config.h | 39 - src/debug.c | 4 +- src/debug.h | 4 +- src/enums.h | 32 - src/gstglbaseaudiovisualizer.c | 486 +++++++++--- src/gstglbaseaudiovisualizer.h | 77 +- src/gstpmaudiovisualizer.c | 1230 +++++++++++++++++++++++++++++ src/gstpmaudiovisualizer.h | 167 ++++ src/gstprojectm.c | 182 +++++ src/{plugin.h => gstprojectm.h} | 31 +- src/gstprojectmbase.c | 690 ++++++++++++++++ src/gstprojectmbase.h | 200 +++++ src/gstprojectmcaps.c | 27 + src/{caps.h => gstprojectmcaps.h} | 10 +- src/gstprojectmconfig.h | 20 + src/plugin.c | 539 ------------- src/projectm.c | 127 --- src/projectm.h | 24 - src/register.c | 38 + src/renderbuffer.c | 653 +++++++++++++++ src/renderbuffer.h | 430 ++++++++++ 27 files changed, 4492 insertions(+), 962 deletions(-) create mode 100644 example/CMakeLists.txt create mode 100644 example/dynpads.c delete mode 100644 src/caps.c delete mode 100644 src/config.h delete mode 100644 src/enums.h create mode 100644 src/gstpmaudiovisualizer.c create mode 100644 src/gstpmaudiovisualizer.h create mode 100644 src/gstprojectm.c rename src/{plugin.h => gstprojectm.h} (66%) create mode 100644 src/gstprojectmbase.c create mode 100644 src/gstprojectmbase.h create mode 100644 src/gstprojectmcaps.c rename src/{caps.h => gstprojectmcaps.h} (55%) create mode 100644 src/gstprojectmconfig.h delete mode 100644 src/plugin.c delete mode 100644 src/projectm.c delete mode 100644 src/projectm.h create mode 100644 src/register.c create mode 100644 src/renderbuffer.c create mode 100644 src/renderbuffer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index e41f87f..2ff14d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,18 +15,22 @@ find_package(GStreamer REQUIRED COMPONENTS gstreamer-audio gstreamer-gl gstreame find_package(GLIB2 REQUIRED) add_library(gstprojectm SHARED - src/caps.h - src/caps.c + src/gstprojectmcaps.h + src/gstprojectmcaps.c src/debug.h src/debug.c - src/config.h - src/enums.h - src/plugin.h - src/plugin.c - src/projectm.h - src/projectm.c + src/gstprojectmconfig.h + src/gstprojectm.h + src/gstprojectm.c src/gstglbaseaudiovisualizer.h src/gstglbaseaudiovisualizer.c + src/gstpmaudiovisualizer.h + src/gstpmaudiovisualizer.c + src/gstprojectmbase.h + src/gstprojectmbase.c + src/register.c + src/renderbuffer.h + src/renderbuffer.c ) target_include_directories(gstprojectm @@ -73,3 +77,5 @@ target_link_libraries(gstprojectm ${GLIB2_LIBRARIES} ${GLIB2_GOBJECT_LIBRARIES} ) + +add_subdirectory(example) diff --git a/README.md b/README.md index 1b43b93..38d85d3 100644 --- a/README.md +++ b/README.md @@ -57,25 +57,50 @@ The documentation has been organized into distinct files, each dedicated to a sp - **[OSX](docs/OSX.md)** - **[Windows](docs/WINDOWS.md)** -Once the plugin has been installed, you can use it something like this: +Once the plugin has been installed, you can use it something like this to render in real-time to an OpenGL window: ```shell -gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false +gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false ``` -Or to convert an audio file to video: +To render from a live source in real-time to a gl window, an identity element can be used to provide a proper timestamp source for the pipeline. This example also includes a texture directory: +```shell +gst-launch souphttpsrc location=http://your-radio-stream is-live=true ! queue ! decodebin ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! identity single-segment=true sync=true ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 texture-dir=/usr/local/share/projectM/presets-milkdrop-texture-pack ! video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! glimagesink sync=false +``` + +Or to convert an audio file to video using offline rendering: ```shell +gst-launch-1.0 -e \ filesrc location=input.mp3 ! decodebin name=dec \ decodebin ! tee name=t \ t. ! queue ! audioconvert ! audioresample ! \ capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \ - t. ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ - identity sync=false ! videoconvert ! videorate ! video/x-raw,framerate=60/1,width=3840,height=2160 ! \ + t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \ + projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ + identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \ + gldownload \ x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ mp4mux name=mux ! filesink location=render.mp4; ``` +Or converting an audio file with the nVidia optimized encoder, directly from GL memory: +```shell +gst-launch-1.0 -e \ + filesrc location=input.mp3 ! \ + decodebin ! tee name=t \ + t. ! queue ! audioconvert ! audioresample ! \ + capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \ + avenc_aac bitrate=320000 ! queue ! mux. \ + t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! projectm \ + preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ + identity sync=false ! videoconvert ! videorate ! \ + video/x-raw\(memory:GLMemory\),framerate=60/1,width=1920,height=1080 ! \ + nvh264enc ! h264parse ! \ + video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ + mp4mux name=mux ! filesink location=render.mp4; +``` + Available options ```shell @@ -193,21 +218,23 @@ If you have your own ProjectM preset files: Once the plugin has been installed, you can use it something like this: ```shell -gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false +gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false ``` Or to convert an audio file to video: ```shell gst-launch-1.0 -e \ - filesrc location=input.mp3 ! \ + filesrc location=input.mp3 ! decodebin name=dec \ decodebin ! tee name=t \ t. ! queue ! audioconvert ! audioresample ! \ - capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=320000 ! queue ! mux. \ - t. ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets texture-dir=/usr/local/share/projectM/textures preset-duration=6 mesh-size=1024,576 ! \ - identity sync=false ! videoconvert ! videorate ! video/x-raw,framerate=60/1,width=3840,height=2160 ! \ - x264enc bitrate=50000 key-int-max=200 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ - mp4mux name=mux ! filesink location=output.mp4 + capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \ + t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \ + projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ + identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \ + gldownload \ + x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ + mp4mux name=mux ! filesink location=render.mp4; ``` You may need to adjust some elements which may or may not be present in your GStreamer installation, such as x264enc, avenc_aac, etc. @@ -220,6 +247,50 @@ gst-inspect projectm

(back to top)

+## Technical Details + +### 🖼️ OpenGL Rendering and Buffer Handling + +- projectM output is rendered to OpenGL textures via **Frame Buffer Object (FBO)**. +- **Textures are pooled** and reused across frames to avoid excessive GPU memory allocation and de-allocation. +- Each rendered texture becomes a GStreamer video buffer pushed downstream. **All video buffers stay in GPU memory**. + +--- + +### ⏱️ Timing and Synchronization + +The plugin synchronizes rendering to the GStreamer pipeline clock using **audio PTS (presentation timestamp) as the leading reference**. + +Pipeline caps control the desired video framerate for rendering. The render loop is **push-based** to conform with +GStreamer's pipeline timing concept, and to enable faster-than-real-time rendering. +A **fixed number of audio samples is consumed per video frame**. + +**Example:** `735 samples per frame at 44.1 kHz = ~60 FPS.` + + +In real-time pipelines, frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with +pipeline caps fps. + +Video frame PTS offset is derived from the **first audio buffer PTS** or **segment event** plus accumulated samples to align with audio timing. + + +| Timing Source | Source | Applies to clock | Purpose | +|----------------------------------|--------------------|------------------|--------------------------------------------------------------------------------------------| +| Audio Timestamps | Audio Input | Always | Determine video timing and sync. | +| Sample Rate / Pipeline FPS | Audio Input / Caps | Always | Defines how many audio samples are used per frame and target FPS. | +| Segment Info | Segment Event | Always | Tracks running time and playback position. Used for PTS offsets. | +| QoS Feedback | QoS Event | Real-time | Skips outdated frames to reduce latency. | +| Render Frame Drop | Render Loop | Real-time | Drop frames that cannot be rendered in time. | +| Exponential Moving Average (EMA) | Render Loop | Real-time | Adjust plugin target fps when frame render time exceeds real-time budget most of the time. | + + +--- + + +

(back to top)

+ +--- + ## Contributing @@ -261,6 +332,8 @@ Blaquewithaq (Discord: SoFloppy#1289) - [@anomievision](https://twitter.com/anom Mischa (Discord: mish) - [@revmischa](https://github.com/revmischa) +Michael [@mbaetgen-wup](https://github.com/mbaetgen-wup) - michael -at- widerup.com +

(back to top)

diff --git a/convert.sh b/convert.sh index 79bff18..c709008 100644 --- a/convert.sh +++ b/convert.sh @@ -154,13 +154,15 @@ gst-launch-1.0 -e \ t. ! queue ! audioconvert ! audioresample ! \ capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \ avenc_aac bitrate=320000 ! queue ! mux. \ - t. ! queue ! audioconvert ! projectm \ + t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \ + projectm \ preset=$PRESET_PATH \ texture-dir=$TEXTURE_DIR \ preset-duration=$PRESET_DURATION \ mesh-size=${MESH_X},${MESH_Y} ! \ identity sync=false ! videoconvert ! videorate ! \ - video/x-raw,framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \ + video/x-raw\(memory:GLMemory\),framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \ + gldownload ! \ x264enc bitrate=$(($BITRATE * 1000)) key-int-max=200 speed-preset=$SPEED_PRESET ! \ video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ mp4mux name=mux ! filesink location=$OUTPUT_FILE & diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt new file mode 100644 index 0000000..f502740 --- /dev/null +++ b/example/CMakeLists.txt @@ -0,0 +1,30 @@ + +add_executable(dyn-pads-example + dynpads.c +) + +target_include_directories(dyn-pads-example + PUBLIC + ${GSTREAMER_INCLUDE_DIRS} + ${GSTREAMER_BASE_INCLUDE_DIRS} + ${GSTREAMER_AUDIO_INCLUDE_DIRS} + ${GSTREAMER_GL_INCLUDE_DIRS} + ${GLIB2_INCLUDE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_link_libraries(dyn-pads-example + PRIVATE + libprojectM::projectM + libprojectM::playlist + PUBLIC + ${GSTREAMER_LIBRARIES} + ${GSTREAMER_BASE_LIBRARIES} + ${GSTREAMER_AUDIO_LIBRARIES} + ${GSTREAMER_VIDEO_LIBRARIES} + ${GSTREAMER_GL_LIBRARIES} + ${GSTREAMER_PBUTILS_LIBRARIES} + ${GLIB2_LIBRARIES} + ${GLIB2_GOBJECT_LIBRARIES} + gstprojectm +) diff --git a/example/dynpads.c b/example/dynpads.c new file mode 100644 index 0000000..b388c5e --- /dev/null +++ b/example/dynpads.c @@ -0,0 +1,238 @@ +#include + +#include + +/** + * Example for a "pad added" signal callback handler for handling gst + * demuxer-like elements. + * + * @param element Callback param for the gst element receiving the event. + * @param new_pad The pad being added. + * @param data The gst element adding the pad (e.g. demuxer). + */ +static void on_pad_added(GstElement *element, GstPad *new_pad, gpointer data) { + + GstPad *sink_pad; + GstElement *downstream_element = GST_ELEMENT(data); + GstPadLinkReturn ret; + GstCaps *new_pad_caps = NULL; + GstStructure *new_pad_struct = NULL; + const gchar *new_pad_type = NULL; + + g_print("Received new pad '%s' from '%s':\n", GST_PAD_NAME(new_pad), + GST_ELEMENT_NAME(element)); + + /* Check the new pad's capabilities to determine its media type */ + new_pad_caps = gst_pad_get_current_caps(new_pad); + new_pad_struct = gst_caps_get_structure(new_pad_caps, 0); + new_pad_type = gst_structure_get_name(new_pad_struct); + + /* Get the sink pad from the downstream element (either audio or video queue) + */ + if (g_str_has_prefix(new_pad_type, "video/x-raw")) { + sink_pad = gst_element_get_static_pad(downstream_element, "sink"); + } else if (g_str_has_prefix(new_pad_type, "audio/x-raw")) { + sink_pad = gst_element_get_static_pad(downstream_element, "sink"); + } else { + g_print(" It has type '%s', which we don't handle. Ignoring.\n", + new_pad_type); + goto exit; + } + + /* Check if the pads are already linked */ + if (gst_pad_is_linked(sink_pad)) { + g_print(" We already linked pad %s. Ignoring.\n", GST_PAD_NAME(new_pad)); + goto exit; + } + + /* Link the new pad to the sink pad */ + ret = gst_pad_link(new_pad, sink_pad); + if (GST_PAD_LINK_FAILED(ret)) { + g_print(" Type is '%s' but link failed.\n", new_pad_type); + } else { + g_print(" Link succeeded (type '%s').\n", new_pad_type); + } + +exit: + /* Clean up */ + if (new_pad_caps != NULL) + gst_caps_unref(new_pad_caps); + + if (sink_pad != NULL) + gst_object_unref(sink_pad); +} + +/** + * Main function to build and run the pipeline to consume a live audio stream + * and render projectM to an OpenGL window in real-time. + * + * souphttpsrc location=... is-live=true ! queue ! decodebin ! audioconvert ! + * "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! + * projectm preset=... preset-duration=... mesh-size=48,32 texture-dir=... ! + * video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! queue + * leaky=downstream max-size-buffers=1 ! glimagesink sync=true + */ +int main(int argc, char *argv[]) { + GstElement *source, *demuxer, *queue, *audioconvert, *audio_capsfilter, + *identity, *projectm_plugin, *video_capsfilter, *sync_queue, *sink; + GstBus *bus; + GstElement *pipeline; + + gst_init(&argc, &argv); + + // make audio caps + GstCaps *audio_caps = + gst_caps_new_simple("audio/x-raw", "format", G_TYPE_STRING, "S16LE", + "rate", G_TYPE_INT, 44100, "channels", G_TYPE_INT, 2, + "layout", G_TYPE_STRING, "interleaved", NULL); + + // make video caps + // todo: adjust caps as desired, keep in mind the hardware needs to be able to + // keep up in order for this plugin to work flawlessly. + GstCaps *video_caps = gst_caps_new_simple( + "video/x-raw", "format", G_TYPE_STRING, "RGBA", "width", G_TYPE_INT, 1920, + "height", G_TYPE_INT, 1080, "framerate", GST_TYPE_FRACTION, 60, 1, NULL); + + // Create the GL memory feature set. + GstCapsFeatures *features = + gst_caps_features_new_single(GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + + // Add the GL memory feature set to the structure. + gst_caps_set_features(video_caps, 0, features); + + // Create pipeline elements + source = gst_element_factory_make("souphttpsrc", "source"); + g_object_set(source, + // todo: configure your stream here.. + "location", "http://your-stream-url", "is-live", TRUE, NULL); + + // basic stream buffering + queue = gst_element_factory_make("queue", "queue"); + + // decodebin to decode the stream audio format + demuxer = gst_element_factory_make("decodebin", "demuxer"); + g_object_set(G_OBJECT(demuxer), "max-size-time", "100000000", NULL); + + // convert the audio stream to something we can understand (if needed) + audioconvert = gst_element_factory_make("audioconvert", "audioconvert"); + + // tell pipeline which audio format we need + audio_capsfilter = gst_element_factory_make("capsfilter", "audio_capsfilter"); + g_object_set(G_OBJECT(audio_capsfilter), "caps", audio_caps, NULL); + + // create an identity element to provide a sream clock, since we won't get one + // from souphttpsrc + identity = gst_element_factory_make("identity", "identity"); + g_object_set(G_OBJECT(identity), "single-segment", TRUE, "sync", TRUE, NULL); + + // configure projectM plugin + projectm_plugin = gst_element_factory_make("projectm", "projectm"); + + // todo: configure your settings here.. + g_object_set(G_OBJECT(projectm_plugin), "preset-duration", 10.0, + //"preset", "/your/presets/directory", + "mesh-size", "48,32", + //"texture-dir", "/your/presets-milkdrop-texture-pack-directory", + NULL); + + // set video caps we want + video_capsfilter = gst_element_factory_make("capsfilter", "video_capsfilter"); + g_object_set(G_OBJECT(video_capsfilter), "caps", video_caps, NULL); + + // optional: create a queue in front of the glimagesink to throw out buffers + // that piling up in front of rendering just keep the latest one, the others + // will most likely be late + sync_queue = gst_element_factory_make("queue", "sync_queue"); + // 0 (no): The default behavior. The queue is not leaky and will block when + // full. 1 (upstream): The queue drops new incoming buffers when it is full. + // 2 (downstream): The queue drops the oldest buffers in the queue when it is + // full. + g_object_set(G_OBJECT(sync_queue), "leaky", 2, "max-size-buffers", 1, NULL); + + // create sink for real-time rendering (synced to the pipeline clock) + sink = gst_element_factory_make("glimagesink", "sink"); + g_object_set(G_OBJECT(sink), "sync", TRUE, NULL); + + pipeline = gst_pipeline_new("test-pipeline"); + + if (!pipeline || !source || !demuxer || !queue || !projectm_plugin || + !video_capsfilter || !sync_queue || !sink) { + g_printerr("One or more elements could not be created. Exiting.\n"); + return -1; + } + + /* Set up the pipeline */ + gst_bin_add_many(GST_BIN(pipeline), source, queue, demuxer, audioconvert, + audio_capsfilter, identity, projectm_plugin, + video_capsfilter, sync_queue, sink, NULL); + + /* Link the elements (but not the demuxer's dynamic pad yet) */ + if (!gst_element_link(source, queue)) { + g_printerr("Elements could not be linked (source to queue). Exiting.\n"); + return -1; + } + if (!gst_element_link(queue, demuxer)) { + g_printerr( + "Elements could not be linked (queue, audioconvert). Exiting.\n"); + return -1; + } + /* not yet! + if (!gst_element_link(demuxer, audioconvert)) { + g_printerr("Elements could not be linked (demuxer, queue). Exiting.\n"); + return -1; + } + */ + if (!gst_element_link(audioconvert, audio_capsfilter)) { + g_printerr("Elements could not be linked (audioconvert to " + "audio_capsfilter). Exiting.\n"); + return -1; + } + if (!gst_element_link(audio_capsfilter, identity)) { + g_printerr("Elements could not be linked (audio_capsfilter to identity). " + "Exiting.\n"); + return -1; + } + if (!gst_element_link(identity, projectm_plugin)) { + g_printerr("Elements could not be linked (identity to projectm_plugin). " + "Exiting.\n"); + return -1; + } + if (!gst_element_link(projectm_plugin, video_capsfilter)) { + g_printerr("Elements could not be linked (projectm_plugin to capsfilter). " + "Exiting.\n"); + return -1; + } + if (!gst_element_link(video_capsfilter, sync_queue)) { + g_printerr("Elements could not be linked (video_capsfilter to sync_queue). " + "Exiting.\n"); + return -1; + } + if (!gst_element_link(sync_queue, sink)) { + g_printerr("Elements could not be linked (sync_queue to sink). Exiting.\n"); + return -1; + } + + gst_caps_unref(video_caps); + gst_caps_unref(audio_caps); + + /* Connect the "pad-added" signal */ + g_signal_connect(demuxer, "pad-added", G_CALLBACK(on_pad_added), + audioconvert); + + /* Set the pipeline to the PLAYING state */ + GMainLoop *loop = g_main_loop_new(NULL, FALSE); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + g_main_loop_run(loop); + + /* Wait until error or EOS */ + bus = gst_element_get_bus(pipeline); + gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, + GST_MESSAGE_ERROR | GST_MESSAGE_EOS); + + /* Clean up */ + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_object_unref(bus); + gst_object_unref(pipeline); + + return 0; +} diff --git a/src/caps.c b/src/caps.c deleted file mode 100644 index 09b5add..0000000 --- a/src/caps.c +++ /dev/null @@ -1,51 +0,0 @@ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include - -#include "caps.h" -#include "plugin.h" - -GST_DEBUG_CATEGORY_STATIC(gst_projectm_caps_debug); -#define GST_CAT_DEFAULT gst_projectm_caps_debug - -const gchar *get_audio_sink_cap(unsigned int type) { - const char *format; - - switch (type) { - case 0: - format = - GST_AUDIO_CAPS_MAKE("audio/x-raw, " - "format = (string) " GST_AUDIO_NE( - S16) ", " - "layout = (string) interleaved, " - "channels = (int) { 2 }, " - "rate = (int) { 44100 }, " - "channel-mask = (bitmask) { 0x0003 }"); - break; - default: - format = NULL; - break; - } - - return format; -} - -const gchar *get_video_src_cap(unsigned int type) { - const char *format; - - switch (type) { - case 0: - format = GST_VIDEO_CAPS_MAKE("video/x-raw, format = (string) { ABGR }, " - "framerate=(fraction)[0/1,MAX]"); - break; - default: - format = NULL; - break; - } - - return format; -} \ No newline at end of file diff --git a/src/config.h b/src/config.h deleted file mode 100644 index bc83e3b..0000000 --- a/src/config.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef __GST_PROJECTM_CONFIG_H__ -#define __GST_PROJECTM_CONFIG_H__ - -#include - -G_BEGIN_DECLS - -/** - * @brief Plugin Details - */ - -#define PACKAGE "GstProjectM" -#define PACKAGE_NAME "GstProjectM" -#define PACKAGE_VERSION "0.0.2" -#define PACKAGE_LICENSE "LGPL" -#define PACKAGE_ORIGIN "https://github.com/projectM-visualizer/gst-projectm" - -/** - * @brief ProjectM Settings (defaults) - */ - -#define DEFAULT_PRESET_PATH NULL -#define DEFAULT_TEXTURE_DIR_PATH NULL -#define DEFAULT_BEAT_SENSITIVITY 1.0 -#define DEFAULT_HARD_CUT_DURATION 3.0 -#define DEFAULT_HARD_CUT_ENABLED FALSE -#define DEFAULT_HARD_CUT_SENSITIVITY 1.0 -#define DEFAULT_SOFT_CUT_DURATION 3.0 -#define DEFAULT_PRESET_DURATION 0.0 -#define DEFAULT_MESH_SIZE "48,32" -#define DEFAULT_ASPECT_CORRECTION TRUE -#define DEFAULT_EASTER_EGG 0.0 -#define DEFAULT_PRESET_LOCKED FALSE -#define DEFAULT_ENABLE_PLAYLIST TRUE -#define DEFAULT_SHUFFLE_PRESETS TRUE // depends on ENABLE_PLAYLIST - -G_END_DECLS - -#endif /* __GST_PROJECTM_CONFIG_H__ */ diff --git a/src/debug.c b/src/debug.c index 9a948aa..f269b7e 100644 --- a/src/debug.c +++ b/src/debug.c @@ -7,7 +7,7 @@ #include "debug.h" -void gl_error_handler(GstGLContext *context, gpointer data) { +void gl_error_handler(GstGLContext *context) { GLuint error = context->gl_vtable->GetError(); switch (error) { @@ -47,4 +47,4 @@ void gl_error_handler(GstGLContext *context, gpointer data) { g_error("OpenGL Error: Unknown error code - 0x%x\n", error); break; } -} \ No newline at end of file +} diff --git a/src/debug.h b/src/debug.h index c0b3969..8096afd 100644 --- a/src/debug.h +++ b/src/debug.h @@ -26,8 +26,8 @@ G_BEGIN_DECLS * @param context The OpenGL context. * @param data Unused. */ -void gl_error_handler(GstGLContext *context, gpointer data); +void gl_error_handler(GstGLContext *context); G_END_DECLS -#endif /* __GST_PROJECTM_DEBUG_H__ */ \ No newline at end of file +#endif /* __GST_PROJECTM_DEBUG_H__ */ diff --git a/src/enums.h b/src/enums.h deleted file mode 100644 index 863d677..0000000 --- a/src/enums.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef __GST_PROJECTM_ENUMS_H__ -#define __GST_PROJECTM_ENUMS_H__ - -#include - -G_BEGIN_DECLS - -/** - * @brief Properties - */ - -enum { - PROP_0, - PROP_PRESET_PATH, - PROP_TEXTURE_DIR_PATH, - PROP_BEAT_SENSITIVITY, - PROP_HARD_CUT_DURATION, - PROP_HARD_CUT_ENABLED, - PROP_HARD_CUT_SENSITIVITY, - PROP_SOFT_CUT_DURATION, - PROP_PRESET_DURATION, - PROP_MESH_SIZE, - PROP_ASPECT_CORRECTION, - PROP_EASTER_EGG, - PROP_PRESET_LOCKED, - PROP_SHUFFLE_PRESETS, - PROP_ENABLE_PLAYLIST -}; - -G_END_DECLS - -#endif /* __GST_PROJECTM_ENUMS_H__ */ diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index e1f786a..df92909 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -35,44 +35,72 @@ #endif #include "gstglbaseaudiovisualizer.h" +#include "gstpmaudiovisualizer.h" +#include "renderbuffer.h" + +#include #include /** * SECTION:GstGLBaseAudioVisualizer - * @short_description: #GstAudioVisualizer subclass for injecting OpenGL + * @short_description: #GstPMAudioVisualizer subclass for injecting OpenGL * resources in a pipeline * @title: GstGLBaseAudioVisualizer - * @see_also: #GstAudioVisualizer + * @see_also: #GstPMAudioVisualizer * - * Wrapper for GstAudioVisualizer for handling OpenGL contexts. + * Wrapper for GstPMAudioVisualizer for handling OpenGL contexts. * * #GstGLBaseAudioVisualizer handles the nitty gritty details of retrieving an * OpenGL context. It also provides `gl_start()` and `gl_stop()` virtual methods * that ensure an OpenGL context is available and current in the calling thread - * for initializing and cleaning up OpenGL dependent resources. The `gl_render` - * virtual method is used to perform OpenGL rendering. + * for initializing and cleaning up OpenGL resources. The `render` + * virtual method of the GstPMAudioVisualizer is implemented to perform OpenGL + * rendering. The implementer provides an implementation for fill_gl_memory to + * render directly to gl memory. + * + * Typical plug-in call order for implementer-provided functions: + * - gl_start (once) + * - setup (every time caps change, typically once) + * - fill_gl_memory (once for each frame) + * - gl_stop (once) */ #define GST_CAT_DEFAULT gst_gl_base_audio_visualizer_debug -GST_DEBUG_CATEGORY_STATIC(GST_CAT_DEFAULT); +GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); + +#define DEFAULT_TIMESTAMP_OFFSET 0 + +/** + * Allow 0.75 * fps frame duration as wait time for frame render queuing. + */ +#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N 3 +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D 4 +#endif + +#define DEFAULT_MIN_FPS_N 1 +#define DEFAULT_MIN_FPS_D 1 struct _GstGLBaseAudioVisualizerPrivate { GstGLContext *other_context; gint64 n_frames; /* total frames sent */ - gboolean gl_result; + gboolean gl_started; GRecMutex context_lock; + GstGLFramebuffer *fbo; + RBRenderBuffer render_buffer; + gboolean is_realtime; }; /* Properties */ -enum { PROP_0 }; +enum { PROP_0, PROP_MIN_FPS_N, PROP_MIN_FPS_D }; #define gst_gl_base_audio_visualizer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE( GstGLBaseAudioVisualizer, gst_gl_base_audio_visualizer, - GST_TYPE_AUDIO_VISUALIZER, + GST_TYPE_PM_AUDIO_VISUALIZER, G_ADD_PRIVATE(GstGLBaseAudioVisualizer) GST_DEBUG_CATEGORY_INIT(gst_gl_base_audio_visualizer_debug, "glbaseaudiovisualizer", 0, @@ -88,39 +116,93 @@ static void gst_gl_base_audio_visualizer_get_property(GObject *object, GValue *value, GParamSpec *pspec); +/** + * Discover gl context / display from gst. + */ static void gst_gl_base_audio_visualizer_set_context(GstElement *element, GstContext *context); +/** + * Handle pipeline state changes. + */ static GstStateChangeReturn gst_gl_base_audio_visualizer_change_state(GstElement *element, GstStateChange transition); -static gboolean gst_gl_base_audio_visualizer_render(GstAudioVisualizer *bscope, - GstBuffer *audio, - GstVideoFrame *video); +/** + * Renders a video frame using gl, impl for parent class + * GstPMAudioVisualizerClass. + */ +static GstFlowReturn gst_gl_base_audio_visualizer_parent_render( + GstPMAudioVisualizer *bscope, GstBuffer *audio, GstClockTime pts, + GstClockTime running_time, guint64 frame_duration); + +/** + * Internal utility for resetting state on start \ + */ static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav); + +/** + * Internal utility for cleaning up gl context on stop + */ static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav); -static gboolean -gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, - GstQuery *query); +/** + * GL memory pool allocation impl for parent class GstPMAudioVisualizerClass. + */ +static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( + GstPMAudioVisualizer *pmav, GstQuery *query); + +/** + * called when format changes, default empty v-impl for this class. can be + * overwritten by implementer. + */ static gboolean gst_gl_base_audio_visualizer_default_setup(GstGLBaseAudioVisualizer *glav); + +/** + * gl context is started and usable. called from gl thread. default empty v-impl + * for this class, can be overwritten by implementer. + */ static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav); + +/** + * GL context is shutting down. called from gl thread. default empty v-impl for + * this class. can be overwritten by implementer. + */ static void gst_gl_base_audio_visualizer_default_gl_stop(GstGLBaseAudioVisualizer *glav); -static gboolean gst_gl_base_audio_visualizer_default_gl_render( - GstGLBaseAudioVisualizer *glav, GstBuffer *audio, GstVideoFrame *video); +/** + * Default empty v-impl for rendering a frame. called from gl thread. can be + * overwritten by implementer. + */ +static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( + GstAVRenderParams *render_data); + +/** + * Find a valid gl context. lock must have already been acquired. + */ static gboolean gst_gl_base_audio_visualizer_find_gl_context_unlocked( GstGLBaseAudioVisualizer *glav); -static gboolean gst_gl_base_audio_visualizer_setup(GstAudioVisualizer *gstav); +/** + * Called whenever the caps change, src and sink caps are both set. Impl for + * parent class GstPMAudioVisualizerClass. + */ +static gboolean +gst_gl_base_audio_visualizer_parent_setup(GstPMAudioVisualizer *pmav); + +/** + * Called from gl thread: fbo rtt rending function. + */ +static void gst_gl_base_audio_visualizer_fill_gl(GstGLContext *context, + gpointer render_slot_ptr); static void gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { GObjectClass *gobject_class = G_OBJECT_CLASS(klass); - GstAudioVisualizerClass *gstav_class = GST_AUDIO_VISUALIZER_CLASS(klass); + GstPMAudioVisualizerClass *pmav_class = GST_PM_AUDIO_VISUALIZER_CLASS(klass); GstElementClass *element_class = GST_ELEMENT_CLASS(klass); gobject_class->finalize = gst_gl_base_audio_visualizer_finalize; @@ -130,31 +212,77 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { element_class->set_context = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_set_context); - element_class->change_state = + pmav_class->change_state = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_change_state); - gstav_class->decide_allocation = - GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_decide_allocation); - gstav_class->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_setup); + pmav_class->decide_allocation = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_decide_allocation); - gstav_class->render = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_render); + pmav_class->setup = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_setup); + + pmav_class->render = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_parent_render); klass->supported_gl_api = GST_GL_API_ANY; + klass->gl_start = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_start); + klass->gl_stop = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_stop); - klass->gl_render = - GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_gl_render); + klass->setup = GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_setup); + + klass->fill_gl_memory = + GST_DEBUG_FUNCPTR(gst_gl_base_audio_visualizer_default_fill_gl_memory); + + g_object_class_install_property( + gobject_class, PROP_MIN_FPS_N, + g_param_spec_int("min-fps-n", "Min FPS numerator", + "Specifies the numerator for the min fps (EMA)", 1, 1000, + DEFAULT_MIN_FPS_N, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_MIN_FPS_D, + g_param_spec_int("min-fps-d", "Min FPS denominator", + "Specifies the denominator for the min fps (EMA)", 1, + 1000, DEFAULT_MIN_FPS_D, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); +} + +/** + * Callback function to receive fps changes from render buffer. + * + * @param user_data Render buffer to use. + * @param frame_duration New fps frame duration. + */ +static void adjust_fps_callback(gpointer user_data, guint64 frame_duration) { + + if (frame_duration == 0) { + return; + } + + RBRenderBuffer *render_state = (RBRenderBuffer *)user_data; + + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(render_state->plugin); + + gst_pm_audio_visualizer_adjust_fps(scope, frame_duration); } static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv = gst_gl_base_audio_visualizer_get_instance_private(glav); glav->priv->gl_started = FALSE; - glav->priv->gl_result = TRUE; + glav->priv->fbo = NULL; + glav->priv->is_realtime = FALSE; glav->context = NULL; + + glav->min_fps_n = DEFAULT_MIN_FPS_N; + glav->min_fps_d = DEFAULT_MIN_FPS_D; + g_rec_mutex_init(&glav->priv->context_lock); + gst_gl_base_audio_visualizer_start(glav); } @@ -174,6 +302,15 @@ static void gst_gl_base_audio_visualizer_set_property(GObject *object, GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); switch (prop_id) { + + case PROP_MIN_FPS_N: + glav->min_fps_n = g_value_get_int(value); + break; + + case PROP_MIN_FPS_D: + glav->min_fps_d = g_value_get_int(value); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -187,6 +324,15 @@ static void gst_gl_base_audio_visualizer_get_property(GObject *object, GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); switch (prop_id) { + + case PROP_MIN_FPS_N: + g_value_set_int(value, glav->min_fps_n); + break; + + case PROP_MIN_FPS_D: + g_value_set_int(value, glav->min_fps_d); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -212,8 +358,7 @@ static void gst_gl_base_audio_visualizer_set_context(GstElement *element, if (old_display != new_display) { gst_clear_object(&glav->context); if (gst_gl_base_audio_visualizer_find_gl_context_unlocked(glav)) { - // TODO does this need to be handled ? - // gst_pad_mark_reconfigure (GST_BASE_SRC_PAD (glav)); + gst_pad_mark_reconfigure(GST_BASE_SRC_PAD(glav)); } } } @@ -237,6 +382,7 @@ gst_gl_base_audio_visualizer_default_setup(GstGLBaseAudioVisualizer *glav) { static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, gpointer data) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(data); + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(data); GstGLBaseAudioVisualizerClass *glav_class = GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(glav); @@ -244,7 +390,30 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, gst_gl_insert_debug_marker(glav->context, "starting element %s", GST_OBJECT_NAME(glav)); + // init fbo for rtt + glav->priv->fbo = gst_gl_framebuffer_new_with_default_depth( + context, GST_VIDEO_INFO_WIDTH(&pmav->vinfo), + GST_VIDEO_INFO_HEIGHT(&pmav->vinfo)); + + // initialize render buffer + GstClockTime max_frame_duration = + gst_util_uint64_scale_int(GST_SECOND, glav->min_fps_d, glav->min_fps_n); + + GstClockTime caps_frame_duration = + gst_util_uint64_scale_int(GST_SECOND, GST_VIDEO_INFO_FPS_D(&pmav->vinfo), + GST_VIDEO_INFO_FPS_N(&pmav->vinfo)); + + rb_init_render_buffer(&glav->priv->render_buffer, GST_OBJECT(glav), + gst_gl_base_audio_visualizer_fill_gl, + adjust_fps_callback, max_frame_duration, + caps_frame_duration, glav->priv->is_realtime); + + // cascade gl start to implementor glav->priv->gl_started = glav_class->gl_start(glav); + + // get gl rendering going + rb_start_render_thread(&glav->priv->render_buffer, glav->context, + pmav->srcpad); } static void @@ -256,87 +425,219 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, GstGLBaseAudioVisualizerClass *glav_class = GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(glav); - GST_INFO_OBJECT(glav, "stopping"); + GST_INFO_OBJECT(glav, "gl stopping"); gst_gl_insert_debug_marker(glav->context, "stopping element %s", GST_OBJECT_NAME(glav)); - if (glav->priv->gl_started) + // stop gl rendering first + rb_stop_render_thread(&glav->priv->render_buffer); + + // clean up implementor + if (glav->priv->gl_started) { glav_class->gl_stop(glav); + } glav->priv->gl_started = FALSE; -} -static gboolean gst_gl_base_audio_visualizer_setup(GstAudioVisualizer *gstav) { - GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); - GstGLBaseAudioVisualizerClass *glav_class = - GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(gstav); + // clean up render buffer + rb_dispose_render_buffer(&glav->priv->render_buffer); - // cascade setup to the derived plugin after gl initialization has been - // completed - return glav_class->setup(glav); + // clean up state + if (glav->priv->fbo) { + gst_object_unref(glav->priv->fbo); + } } -static gboolean gst_gl_base_audio_visualizer_default_gl_render( - GstGLBaseAudioVisualizer *glav, GstBuffer *audio, GstVideoFrame *video) { +static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( + GstAVRenderParams *render_data) { return TRUE; } -typedef struct { - GstGLBaseAudioVisualizer *glav; - GstBuffer *in_audio; - GstVideoFrame *out_video; -} GstGLRenderCallbackParams; +static void gst_gl_base_audio_visualizer_fill_gl(GstGLContext *context, + gpointer render_slot_ptr) { + + // we're inside the gl thread! + + RBSlot *render_slot = (RBSlot *)render_slot_ptr; + + GstGLBaseAudioVisualizer *glav = + GST_GL_BASE_AUDIO_VISUALIZER(render_slot->plugin); -static void -gst_gl_base_audio_visualizer_gl_thread_render_callback(gpointer params) { - GstGLRenderCallbackParams *cb_params = (GstGLRenderCallbackParams *)params; GstGLBaseAudioVisualizerClass *klass = - GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(cb_params->glav); + GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(render_slot->plugin); + + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(render_slot->plugin); + + GstBuffer *out_buf; + GstVideoFrame out_video; + + // obtain output buffer from the (GL texture backed) pool + gst_pm_audio_visualizer_util_prepare_output_buffer(pmav, &out_buf); + + // Check for GL sync meta + GstGLSyncMeta *sync_meta = gst_buffer_get_gl_sync_meta(out_buf); + + if (sync_meta) { + // wait until GPU is done using this buffer should not be needed + // gst_gl_sync_meta_wait(sync_meta, glav->context); + } + + // GstClockTime after_prepare = gst_util_get_timestamp(); + + // map output video frame to buffer outbuf with gl flags + gst_video_frame_map(&out_video, &pmav->vinfo, out_buf, + GST_MAP_WRITE | GST_MAP_GL | + GST_VIDEO_FRAME_MAP_FLAG_NO_REF); - // inside gl thread: call virtual render function with audio and video - cb_params->glav->priv->gl_result = klass->gl_render( - cb_params->glav, cb_params->in_audio, cb_params->out_video); + // GstClockTime after_map = gst_util_get_timestamp(); + GstAVRenderParams ds_rd; + ds_rd.in_audio = render_slot->in_audio; + ds_rd.mem = GST_GL_MEMORY_CAST(gst_buffer_peek_memory(out_buf, 0)); + ds_rd.fbo = glav->priv->fbo; + ds_rd.pts = render_slot->pts; + ds_rd.plugin = glav; + + GST_TRACE_OBJECT(render_slot->plugin, "filling gl memory %p", ds_rd.mem); + + // call virtual render function with audio and video + render_slot->gl_result = klass->fill_gl_memory(&ds_rd); + + gst_video_frame_unmap(&out_video); + + if (sync_meta) + gst_gl_sync_meta_set_sync_point(sync_meta, glav->context); + + render_slot->out_buf = out_buf; + out_buf = NULL; } -static gboolean gst_gl_base_audio_visualizer_render(GstAudioVisualizer *bscope, - GstBuffer *audio, - GstVideoFrame *video) { - GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(bscope); - GstGLRenderCallbackParams cb_params; - GstGLWindow *window; +static GstFlowReturn gst_gl_base_audio_visualizer_fill( + GstPMAudioVisualizer *bscope, GstGLBaseAudioVisualizer *glav, + GstBuffer *audio, GstClockTime pts, GstClockTime running_time, + guint64 frame_duration) { g_rec_mutex_lock(&glav->priv->context_lock); + if (G_UNLIKELY(!glav->context)) + goto not_negotiated; + + /* 0 framerate and we are at the second frame, eos */ + if (G_UNLIKELY(GST_VIDEO_INFO_FPS_N(&bscope->vinfo) == 0 && + glav->priv->n_frames == 1)) + goto eos; + + // prepare args for queuing frame rendering + RBQueueArgs args; + args.render_buffer = &glav->priv->render_buffer; + args.in_audio = audio; + args.pts = pts; + args.frame_duration = frame_duration; + args.latency = bscope->latency; + args.running_time = running_time; + + if (glav->priv->is_realtime == FALSE) { + // wait for each frame to complete + args.sync_rendering = TRUE; + + // unlimited for offline rendering, frames will never be dropped by QoS. + args.max_wait = GST_CLOCK_TIME_NONE; + } else { + // fire and forget, mapping n samples per frame from upstream keeps us in + // sync + args.sync_rendering = FALSE; + + // limit wait based on fps factor, make sure we never wait too long in order + // to keep in sync + args.max_wait = (GstClockTimeDiff)gst_util_uint64_scale_int( + frame_duration, MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N, + MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D); + } - // wrap params into cb_params struct to pass them to the GL window/thread via - // userdata pointer - cb_params.glav = glav; - cb_params.in_audio = audio; - cb_params.out_video = video; + // dispatch gst_gl_base_audio_visualizer_fill_gl to the gl render buffer, + // rendering is deferred. This may block for a while though. + rb_queue_render_job_warn(&args); - window = gst_gl_context_get_window(glav->context); + glav->priv->n_frames++; - // dispatch render call through the gl thread - // call is blocking, accessing audio and video params from gl thread *should* - // be safe - gst_gl_window_send_message( - window, - GST_GL_WINDOW_CB(gst_gl_base_audio_visualizer_gl_thread_render_callback), - &cb_params); + g_rec_mutex_unlock(&glav->priv->context_lock); - gst_object_unref(window); + return GST_FLOW_OK; +not_negotiated: { + g_rec_mutex_unlock(&glav->priv->context_lock); + GST_ELEMENT_ERROR(glav, CORE, NEGOTIATION, (NULL), + (("format wasn't negotiated before get function"))); + return GST_FLOW_NOT_NEGOTIATED; +} +eos: { g_rec_mutex_unlock(&glav->priv->context_lock); + GST_DEBUG_OBJECT(glav, "eos: 0 framerate, frame %d", + (gint)glav->priv->n_frames); + return GST_FLOW_EOS; +} +} - if (glav->priv->gl_result) { - glav->priv->n_frames++; - } else { - // gl error - GST_ELEMENT_ERROR(glav, RESOURCE, NOT_FOUND, - (("failed to render audio visualizer")), - (("A GL error occurred"))); +/** + * Find out if the pipeline is using a real-time clock. + * + * @param element GST element + * @return TRUE in case the element uses a system clock. + */ +static gboolean is_pipeline_realtime(GstElement *element) { + GstClock *clock = gst_element_get_clock(element); + gboolean is_realtime = FALSE; + + if (clock) { + // Compare to the system clock (used for real-time playback) + is_realtime = GST_IS_SYSTEM_CLOCK(clock); + gst_object_unref(clock); } - return glav->priv->gl_result; + return is_realtime; +} + +static gboolean +gst_gl_base_audio_visualizer_parent_setup(GstPMAudioVisualizer *pmav) { + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(pmav); + GstGLBaseAudioVisualizerClass *glav_class = + GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(pmav); + + // configure QoS for the pipeline, disabled for offline rendering + g_rec_mutex_lock(&glav->priv->context_lock); + + gboolean is_realtime = is_pipeline_realtime(GST_ELEMENT(pmav)); + glav->priv->is_realtime = is_realtime; + + g_rec_mutex_unlock(&glav->priv->context_lock); + + // update render buffer config + rb_set_qos_enabled(&glav->priv->render_buffer, is_realtime); + + GstClockTime caps_frame_duration = + gst_util_uint64_scale_int(GST_SECOND, GST_VIDEO_INFO_FPS_D(&pmav->vinfo), + GST_VIDEO_INFO_FPS_N(&pmav->vinfo)); + rb_set_caps_frame_duration(&glav->priv->render_buffer, caps_frame_duration); + + GST_INFO_OBJECT( + glav, + "GL setup - render config: real-time: %s, caps-frame-duration: " + "%" GST_TIME_FORMAT + ", min-fps: %d/%d, min-fps-duration: %" GST_TIME_FORMAT, + is_realtime ? "true" : "false", GST_TIME_ARGS(caps_frame_duration), + glav->min_fps_n, glav->min_fps_d, + GST_TIME_ARGS(glav->priv->render_buffer.max_frame_duration)); + + // cascade setup to the derived plugin after gl initialization has been + // completed + return glav_class->setup(glav); +} + +static GstFlowReturn gst_gl_base_audio_visualizer_parent_render( + GstPMAudioVisualizer *bscope, GstBuffer *audio, GstClockTime pts, + GstClockTime running_time, guint64 frame_duration) { + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(bscope); + + return gst_gl_base_audio_visualizer_fill(bscope, glav, audio, pts, + running_time, frame_duration); } static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav) { @@ -493,10 +794,9 @@ error: { } } -static gboolean -gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, - GstQuery *query) { - GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(gstav); +static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( + GstPMAudioVisualizer *pmav, GstQuery *query) { + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(pmav); GstGLContext *context; GstBufferPool *pool = NULL; GstStructure *config; @@ -524,7 +824,8 @@ gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, gst_video_info_init(&vinfo); gst_video_info_from_caps(&vinfo, caps); size = vinfo.size; - min = max = 0; + min = 0; + max = 0; update_pool = FALSE; } @@ -536,6 +837,12 @@ gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, } config = gst_buffer_pool_get_config(pool); + // there should be at least 2 textures, so that one is rendered while the + // other one is pushed downstream + // todo: pool size config properties needed ? + if (min < 2) { + min = 2; + } gst_buffer_pool_config_set_params(config, caps, size, min, max); gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_META); if (gst_query_find_allocation_meta(query, GST_GL_SYNC_META_API_TYPE, NULL)) @@ -544,6 +851,9 @@ gst_gl_base_audio_visualizer_decide_allocation(GstAudioVisualizer *gstav, gst_buffer_pool_config_add_option( config, GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META); + gst_buffer_pool_config_add_option( + config, GST_BUFFER_POOL_OPTION_GL_TEXTURE_TARGET_2D); + gst_buffer_pool_set_config(pool, config); if (update_pool) @@ -568,10 +878,6 @@ gst_gl_base_audio_visualizer_change_state(GstElement *element, gst_element_state_get_name(GST_STATE_TRANSITION_CURRENT(transition)), gst_element_state_get_name(GST_STATE_TRANSITION_NEXT(transition))); - ret = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition); - if (ret == GST_STATE_CHANGE_FAILURE) - return ret; - switch (transition) { case GST_STATE_CHANGE_READY_TO_NULL: g_rec_mutex_lock(&glav->priv->context_lock); diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index a48781b..a250c4e 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -32,14 +32,14 @@ #ifndef __GST_GL_BASE_AUDIO_VISUALIZER_H__ #define __GST_GL_BASE_AUDIO_VISUALIZER_H__ +#include "gstpmaudiovisualizer.h" #include -#include -#include #include typedef struct _GstGLBaseAudioVisualizer GstGLBaseAudioVisualizer; typedef struct _GstGLBaseAudioVisualizerClass GstGLBaseAudioVisualizerClass; typedef struct _GstGLBaseAudioVisualizerPrivate GstGLBaseAudioVisualizerPrivate; +typedef struct _GstAVRenderParams GstAVRenderParams; G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstGLBaseAudioVisualizer, gst_object_unref) @@ -72,12 +72,22 @@ GType gst_gl_base_audio_visualizer_get_type(void); * The parent instance type of a base GL Audio Visualizer. */ struct _GstGLBaseAudioVisualizer { - GstAudioVisualizer parent; + GstPMAudioVisualizer parent; /*< public >*/ GstGLDisplay *display; GstGLContext *context; + /** + * Minimum FPS numerator setting for EMA. + */ + gint min_fps_n; + + /** + * Minimum FPS denominator setting for EMA. + */ + gint min_fps_d; + /*< private >*/ gpointer _padding[GST_PADDING]; @@ -91,25 +101,76 @@ struct _GstGLBaseAudioVisualizer { * @gl_stop: called in the GL thread to clean up the element GL state. * @gl_render: called in the GL thread to fill the current video texture. * @setup: called when the format changes (delegate from - * GstAudioVisualizer.setup) + * GstPMAudioVisualizer.setup) * * The base class for OpenGL based audio visualizers. - * + * Extends GstPMAudioVisualizer to add GL rendering callbacks. + * Handles GL context and render buffers. */ struct _GstGLBaseAudioVisualizerClass { - GstAudioVisualizerClass parent_class; + GstPMAudioVisualizerClass parent_class; /*< public >*/ + /** + * Supported OpenGL API flags. + */ GstGLAPI supported_gl_api; + + /** + * Virtual function called from gl thread once the gl context can be used for + * initializing gl resources. + */ gboolean (*gl_start)(GstGLBaseAudioVisualizer *glav); + + /** + * Virtual function called from gl thread when gl context is being closed for + * gl resource clean up. + */ void (*gl_stop)(GstGLBaseAudioVisualizer *glav); - gboolean (*gl_render)(GstGLBaseAudioVisualizer *glav, GstBuffer *audio, - GstVideoFrame *video); + + /** + * Virtual function called when caps have been set for the pipeline. + */ gboolean (*setup)(GstGLBaseAudioVisualizer *glav); + + /* Virtual function called to render each frame, in_audio is optional. */ + gboolean (*fill_gl_memory)(GstAVRenderParams *render_data); + /*< private >*/ gpointer _padding[GST_PADDING]; }; +/** + * Parameter struct for rendering calls. + */ +struct _GstAVRenderParams { + + /** + * Context plugin. + */ + GstGLBaseAudioVisualizer *plugin; + + /** + * Framebuffer to use for rendering. + */ + GstGLFramebuffer *fbo; + + /** + * Rendering target GL memory. + */ + GstGLMemory *mem; + + /** + * Audio data for frame. + */ + GstBuffer *in_audio; + + /** + * Current buffer presentation timestamp. + */ + GstClockTime pts; +}; + G_END_DECLS #endif /* __GST_GL_BASE_AUDIO_VISUALIZER_H__ */ diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c new file mode 100644 index 0000000..0cae8b7 --- /dev/null +++ b/src/gstpmaudiovisualizer.c @@ -0,0 +1,1230 @@ +/* GStreamer + * Copyright (C) <2011> Stefan Kost + * Copyright (C) <2015> Luis de Bethencourt + * + * gstaudiovisualizer.h: base class for audio visualisation elements + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +/** + * SECTION:gstaudiovisualizer + * @title: GstPMAudioVisualizer + * @short_description: Base class for visualizers. + * + * A baseclass for scopes (visualizers). It takes care of re-fitting the + * audio-rate to video-rate and handles renegotiation (downstream video size + * changes). + * + * It also provides several background shading effects. These effects are + * applied to a previous picture before the `render()` implementation can draw a + * new frame. + */ + +/* + * The code in this file is based on + * GStreamer / gst-plugins-base, latest version as of 2025/05/29. + * gst-libs/gst/pbutils/gstaudiovisualizer.c Git Repository: + * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/blob/main/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c + * Original copyright notice has been retained at the top of this file. + * + * The code has been modified to improve compatibility with projectM and OpenGL. + * + * - New apis for implementer-provided memory allocation and video frame + * buffer mapping. Used by gl plugins for mapping video frames directly to gl + * memory. + * + * - Main memory based video frame buffers have been removed. + * + * - Cpu based transition shaders have been removed. + * + * - Bugfix for the amount of bytes being flushed for a single video frame from + * the audio input buffer. + * + * - Uses a sample count based approach for pts/dts timestamps instead + * GstAdapter derived timestamps. + * + * - Consistent locking and fixes for some race conditions. + * + * - Allow dynamic fps adjustments while staying sample accurate. + * + * - Segment event propagation. + * + * - All memory management and rendering is implementer-provided. + * + * Typical plug-in call order for implementer-provided functions: + * - decide_allocation (once) + * - setup (when caps change, typically once) + * - render (once for each frame) + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include + +#include "gstpmaudiovisualizer.h" +#include + +GST_DEBUG_CATEGORY_STATIC(pm_audio_visualizer_debug); +#define GST_CAT_DEFAULT (pm_audio_visualizer_debug) + +/** + * Ignore QoS events during the first couple of frames that can cause a start + * delay. + */ +#ifndef QOS_IGNORE_FIRST_N_FRAMES +#define QOS_IGNORE_FIRST_N_FRAMES 5 +#endif + +/** + * Min latency change required to push a latency event + * upstream. The latency is compared to last published latency. + */ +#ifndef LATENCY_EVENT_MIN_CHANGE +#define LATENCY_EVENT_MIN_CHANGE GST_MSECOND +#endif + +enum { PROP_0 }; + +static GstBaseTransformClass *parent_class = NULL; +static gint private_offset = 0; + +static void +gst_pm_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass); +static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, + GstPMAudioVisualizerClass *g_class); +static void gst_pm_audio_visualizer_set_property(GObject *object, guint prop_id, + const GValue *value, + GParamSpec *pspec); +static void gst_pm_audio_visualizer_get_property(GObject *object, guint prop_id, + GValue *value, + GParamSpec *pspec); +static void gst_pm_audio_visualizer_dispose(GObject *object); + +static gboolean +gst_pm_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope); +static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps); +static gboolean +gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps); + +static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, + GstObject *parent, + GstBuffer *buffer); + +static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, + GstObject *parent, + GstEvent *event); +static gboolean gst_pm_audio_visualizer_sink_event(GstPad *pad, + GstObject *parent, + GstEvent *event); + +static gboolean gst_pm_audio_visualizer_src_query(GstPad *pad, + GstObject *parent, + GstQuery *query); + +static GstStateChangeReturn +gst_pm_audio_visualizer_parent_change_state(GstElement *element, + GstStateChange transition); + +static GstStateChangeReturn +gst_pm_audio_visualizer_default_change_state(GstElement *element, + GstStateChange transition); + +static gboolean +gst_pm_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, + GstCaps *outcaps); + +static gboolean +gst_pm_audio_visualizer_default_decide_allocation(GstPMAudioVisualizer *scope, + GstQuery *query); + +static void gst_pm_audio_visualizer_send_latency_if_needed_unlocked( + GstPMAudioVisualizer *scope); + +struct _GstPMAudioVisualizerPrivate { + gboolean negotiated; + + GstBufferPool *pool; + gboolean pool_active; + GstAllocator *allocator; + GstAllocationParams params; + GstQuery *query; + + /* pads */ + GstPad *sinkpad; + + GstAdapter *adapter; + + GstBuffer *inbuf; + + guint spf; /* samples per video frame */ + + /* QoS stuff */ /* with LOCK */ + gdouble proportion; + /* qos: earliest time to render the next frame, the render loop will skip + * frames until this time */ + GstClockTime earliest_time; + + guint dropped; /* frames dropped / not dropped */ + guint processed; + + /* samples consumed, relative to the current segment. Basis for timestamps. */ + guint64 samples_consumed; + + /* configuration mutex */ + GMutex config_lock; + + GstSegment segment; + + /* ready flag and condition triggered once the plugin is ready to process + * buffers, triggers every time a caps event is processed */ + GCond ready_cond; + gboolean ready; + + /* have src caps been setup */ + gboolean src_ready; + + /* have sink caps been setup */ + gboolean sink_ready; + + /* clock timestamp pts offset, either from first audio buffer pts or segment + * event */ + gboolean pts_offset_initialized; + GstClockTime pts_offset; + GstClockTime caps_frame_duration; + GstClockTime last_reported_latency; + gboolean fps_changed; +}; + +/* base class */ + +GType gst_pm_audio_visualizer_get_type(void) { + static gsize audio_visualizer_type = 0; + + if (g_once_init_enter(&audio_visualizer_type)) { + static const GTypeInfo audio_visualizer_info = { + sizeof(GstPMAudioVisualizerClass), + NULL, + NULL, + (GClassInitFunc)gst_pm_audio_visualizer_class_init, + NULL, + NULL, + sizeof(GstPMAudioVisualizer), + 0, + (GInstanceInitFunc)gst_pm_audio_visualizer_init, + }; + GType _type; + + /* TODO: rename when exporting it as a library */ + _type = + g_type_register_static(GST_TYPE_ELEMENT, "GstPMAudioVisualizer", + &audio_visualizer_info, G_TYPE_FLAG_ABSTRACT); + + private_offset = + g_type_add_instance_private(_type, sizeof(GstPMAudioVisualizerPrivate)); + + g_once_init_leave(&audio_visualizer_type, _type); + } + return (GType)audio_visualizer_type; +} + +static inline GstPMAudioVisualizerPrivate * +gst_pm_audio_visualizer_get_instance_private(GstPMAudioVisualizer *self) { + return (G_STRUCT_MEMBER_P(self, private_offset)); +} + +static void +gst_pm_audio_visualizer_class_init(GstPMAudioVisualizerClass *klass) { + GObjectClass *gobject_class = (GObjectClass *)klass; + GstElementClass *element_class = (GstElementClass *)klass; + + if (private_offset != 0) + g_type_class_adjust_private_offset(klass, &private_offset); + + parent_class = g_type_class_peek_parent(klass); + + GST_DEBUG_CATEGORY_INIT(pm_audio_visualizer_debug, "pmaudiovisualizer", 0, + "projectm audio visualisation base class"); + + gobject_class->set_property = gst_pm_audio_visualizer_set_property; + gobject_class->get_property = gst_pm_audio_visualizer_get_property; + gobject_class->dispose = gst_pm_audio_visualizer_dispose; + + element_class->change_state = + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_parent_change_state); + + klass->change_state = + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_default_change_state); + + klass->decide_allocation = + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_default_decide_allocation); + + klass->segment_change = NULL; +} + +static void gst_pm_audio_visualizer_init(GstPMAudioVisualizer *scope, + GstPMAudioVisualizerClass *g_class) { + GstPadTemplate *pad_template; + + scope->priv = gst_pm_audio_visualizer_get_instance_private(scope); + + /* create the sink pad */ + pad_template = + gst_element_class_get_pad_template(GST_ELEMENT_CLASS(g_class), "sink"); + g_return_if_fail(pad_template != NULL); + scope->priv->sinkpad = gst_pad_new_from_template(pad_template, "sink"); + gst_pad_set_chain_function(scope->priv->sinkpad, + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_chain)); + gst_pad_set_event_function( + scope->priv->sinkpad, + GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_sink_event)); + gst_element_add_pad(GST_ELEMENT(scope), scope->priv->sinkpad); + + /* create the src pad */ + pad_template = + gst_element_class_get_pad_template(GST_ELEMENT_CLASS(g_class), "src"); + g_return_if_fail(pad_template != NULL); + scope->srcpad = gst_pad_new_from_template(pad_template, "src"); + gst_pad_set_event_function( + scope->srcpad, GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_src_event)); + gst_pad_set_query_function( + scope->srcpad, GST_DEBUG_FUNCPTR(gst_pm_audio_visualizer_src_query)); + gst_element_add_pad(GST_ELEMENT(scope), scope->srcpad); + + scope->priv->adapter = gst_adapter_new(); + scope->priv->inbuf = gst_buffer_new(); + g_cond_init(&scope->priv->ready_cond); + + scope->priv->dropped = 0; + scope->priv->earliest_time = 0; + scope->priv->processed = 0; + scope->priv->samples_consumed = 0; + scope->priv->src_ready = FALSE; + scope->priv->sink_ready = FALSE; + scope->priv->ready = FALSE; + scope->priv->pts_offset_initialized = FALSE; + scope->priv->pts_offset = GST_CLOCK_TIME_NONE; + scope->priv->caps_frame_duration = 0; + scope->priv->last_reported_latency = GST_CLOCK_TIME_NONE; + scope->priv->fps_changed = FALSE; + scope->latency = GST_CLOCK_TIME_NONE; + + /* properties */ + + /* reset the initial video state */ + gst_video_info_init(&scope->vinfo); + scope->req_frame_duration = GST_CLOCK_TIME_NONE; + + /* reset the initial state */ + gst_audio_info_init(&scope->ainfo); + gst_video_info_init(&scope->vinfo); + + g_mutex_init(&scope->priv->config_lock); +} + +static void gst_pm_audio_visualizer_set_property(GObject *object, guint prop_id, + const GValue *value, + GParamSpec *pspec) { + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_pm_audio_visualizer_get_property(GObject *object, guint prop_id, + GValue *value, + GParamSpec *pspec) { + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_pm_audio_visualizer_dispose(GObject *object) { + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + + if (scope->priv->adapter) { + g_object_unref(scope->priv->adapter); + scope->priv->adapter = NULL; + } + if (scope->priv->config_lock.p) { + g_mutex_clear(&scope->priv->config_lock); + scope->priv->config_lock.p = NULL; + } + if (scope->priv->ready_cond.p) { + g_cond_clear(&scope->priv->ready_cond); + scope->priv->ready_cond.p = NULL; + } + + G_OBJECT_CLASS(parent_class)->dispose(object); +} + +static void +gst_pm_audio_visualizer_reset_unlocked(GstPMAudioVisualizer *scope) { + + gst_adapter_clear(scope->priv->adapter); + gst_segment_init(&scope->priv->segment, GST_FORMAT_UNDEFINED); + + scope->priv->proportion = 1.0; + scope->priv->earliest_time = 0; + scope->priv->dropped = 0; + scope->priv->processed = 0; + scope->priv->samples_consumed = 0; + scope->priv->pts_offset_initialized = FALSE; + scope->priv->pts_offset = GST_CLOCK_TIME_NONE; + scope->latency = GST_CLOCK_TIME_NONE; +} + +/* */ +static gboolean gst_pm_audio_visualizer_do_setup(GstPMAudioVisualizer *scope) { + + GstPMAudioVisualizerClass *klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); + + GST_OBJECT_LOCK(scope); + scope->priv->earliest_time = 0; + GST_OBJECT_UNLOCK(scope); + + g_mutex_lock(&scope->priv->config_lock); + + scope->priv->spf = gst_util_uint64_scale_int( + GST_AUDIO_INFO_RATE(&scope->ainfo), GST_VIDEO_INFO_FPS_D(&scope->vinfo), + GST_VIDEO_INFO_FPS_N(&scope->vinfo)); + scope->req_spf = scope->priv->spf; + + g_mutex_unlock(&scope->priv->config_lock); + + if (klass->setup && !klass->setup(scope)) + return FALSE; + + GST_INFO_OBJECT( + scope, "video: dimension %dx%d, framerate %d/%d", + GST_VIDEO_INFO_WIDTH(&scope->vinfo), GST_VIDEO_INFO_HEIGHT(&scope->vinfo), + GST_VIDEO_INFO_FPS_N(&scope->vinfo), GST_VIDEO_INFO_FPS_D(&scope->vinfo)); + + GST_INFO_OBJECT(scope, "audio: rate %d, channels: %d, bpf: %d", + GST_AUDIO_INFO_RATE(&scope->ainfo), + GST_AUDIO_INFO_CHANNELS(&scope->ainfo), + GST_AUDIO_INFO_BPF(&scope->ainfo)); + + GST_INFO_OBJECT(scope, "blocks: spf %u, req_spf %u", scope->priv->spf, + scope->req_spf); + + g_mutex_lock(&scope->priv->config_lock); + scope->priv->ready = TRUE; + g_cond_broadcast(&scope->priv->ready_cond); + g_mutex_unlock(&scope->priv->config_lock); + + return TRUE; +} + +static gboolean +gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps) { + GstAudioInfo info; + + if (!gst_audio_info_from_caps(&info, caps)) + goto wrong_caps; + + g_mutex_lock(&scope->priv->config_lock); + scope->ainfo = info; + g_mutex_unlock(&scope->priv->config_lock); + + GST_DEBUG_OBJECT(scope, "audio: channels %d, rate %d", + GST_AUDIO_INFO_CHANNELS(&info), GST_AUDIO_INFO_RATE(&info)); + + if (!gst_pm_audio_visualizer_src_negotiate(scope)) { + goto not_negotiated; + } + + g_mutex_lock(&scope->priv->config_lock); + scope->priv->sink_ready = TRUE; + g_mutex_unlock(&scope->priv->config_lock); + + if (scope->priv->src_ready) { + gst_pm_audio_visualizer_do_setup(scope); + } + + return TRUE; + + /* Errors */ +wrong_caps: { + GST_WARNING_OBJECT(scope, "could not parse caps"); + return FALSE; +} +not_negotiated: { + GST_WARNING_OBJECT(scope, "failed to negotiate"); + return FALSE; +} +} + +static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, + GstCaps *caps) { + GstVideoInfo info; + gboolean res; + + if (!gst_video_info_from_caps(&info, caps)) + goto wrong_caps; + + g_mutex_lock(&scope->priv->config_lock); + + scope->vinfo = info; + + scope->priv->caps_frame_duration = gst_util_uint64_scale_int( + GST_SECOND, GST_VIDEO_INFO_FPS_D(&info), GST_VIDEO_INFO_FPS_N(&info)); + + scope->req_frame_duration = scope->priv->caps_frame_duration; + g_mutex_unlock(&scope->priv->config_lock); + + gst_pad_set_caps(scope->srcpad, caps); + + /* find a pool for the negotiated caps now */ + res = gst_pm_audio_visualizer_do_bufferpool(scope, caps); + gst_caps_unref(caps); + + g_mutex_lock(&scope->priv->config_lock); + scope->priv->src_ready = TRUE; + g_mutex_unlock(&scope->priv->config_lock); + if (scope->priv->sink_ready) { + if (!gst_pm_audio_visualizer_do_setup(scope)) { + goto setup_failed; + } + } + + return res; + + /* ERRORS */ +wrong_caps: { + gst_caps_unref(caps); + GST_DEBUG_OBJECT(scope, "error parsing caps"); + return FALSE; +} + +setup_failed: { + GST_WARNING_OBJECT(scope, "failed to set up"); + return FALSE; +} +} + +static gboolean +gst_pm_audio_visualizer_src_negotiate(GstPMAudioVisualizer *scope) { + GstCaps *othercaps, *target; + GstStructure *structure; + GstCaps *templ; + gboolean ret; + + templ = gst_pad_get_pad_template_caps(scope->srcpad); + + GST_DEBUG_OBJECT(scope, "performing negotiation"); + + /* see what the peer can do */ + othercaps = gst_pad_peer_query_caps(scope->srcpad, NULL); + if (othercaps) { + target = gst_caps_intersect(othercaps, templ); + gst_caps_unref(othercaps); + gst_caps_unref(templ); + + if (gst_caps_is_empty(target)) + goto no_format; + + target = gst_caps_truncate(target); + } else { + target = templ; + } + + target = gst_caps_make_writable(target); + structure = gst_caps_get_structure(target, 0); + gst_structure_fixate_field_nearest_int(structure, "width", 320); + gst_structure_fixate_field_nearest_int(structure, "height", 200); + gst_structure_fixate_field_nearest_fraction(structure, "framerate", 25, 1); + if (gst_structure_has_field(structure, "pixel-aspect-ratio")) + gst_structure_fixate_field_nearest_fraction(structure, "pixel-aspect-ratio", + 1, 1); + + target = gst_caps_fixate(target); + + GST_DEBUG_OBJECT(scope, "final caps are %" GST_PTR_FORMAT, target); + + ret = gst_pm_audio_visualizer_src_setcaps(scope, target); + + return ret; + +no_format: { + gst_caps_unref(target); + return FALSE; +} +} + +/* takes ownership of the pool, allocator and query */ +static gboolean gst_pm_audio_visualizer_set_allocation( + GstPMAudioVisualizer *scope, GstBufferPool *pool, GstAllocator *allocator, + const GstAllocationParams *params, GstQuery *query) { + GstAllocator *oldalloc; + GstBufferPool *oldpool; + GstQuery *oldquery; + GstPMAudioVisualizerPrivate *priv = scope->priv; + + GST_OBJECT_LOCK(scope); + oldpool = priv->pool; + priv->pool = pool; + priv->pool_active = FALSE; + + oldalloc = priv->allocator; + priv->allocator = allocator; + + oldquery = priv->query; + priv->query = query; + + if (params) + priv->params = *params; + else + gst_allocation_params_init(&priv->params); + GST_OBJECT_UNLOCK(scope); + + if (oldpool) { + GST_DEBUG_OBJECT(scope, "deactivating old pool %p", oldpool); + gst_buffer_pool_set_active(oldpool, FALSE); + gst_object_unref(oldpool); + } + if (oldalloc) { + gst_object_unref(oldalloc); + } + if (oldquery) { + gst_query_unref(oldquery); + } + return TRUE; +} + +static gboolean +gst_pm_audio_visualizer_do_bufferpool(GstPMAudioVisualizer *scope, + GstCaps *outcaps) { + GstQuery *query; + gboolean result = TRUE; + GstBufferPool *pool = NULL; + GstPMAudioVisualizerClass *klass; + GstAllocator *allocator; + GstAllocationParams params; + + /* not passthrough, we need to allocate */ + /* find a pool for the negotiated caps now */ + GST_DEBUG_OBJECT(scope, "doing allocation query"); + query = gst_query_new_allocation(outcaps, TRUE); + + if (!gst_pad_peer_query(scope->srcpad, query)) { + /* not a problem, we use the query defaults */ + GST_DEBUG_OBJECT(scope, "allocation query failed"); + } + + klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); + + GST_DEBUG_OBJECT(scope, "calling decide_allocation"); + g_assert(klass->decide_allocation != NULL); + result = klass->decide_allocation(scope, query); + + GST_DEBUG_OBJECT(scope, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, + query); + + if (!result) + goto no_decide_allocation; + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params(query) > 0) { + gst_query_parse_nth_allocation_param(query, 0, &allocator, ¶ms); + } else { + allocator = NULL; + gst_allocation_params_init(¶ms); + } + + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + + /* now store */ + result = gst_pm_audio_visualizer_set_allocation(scope, pool, allocator, + ¶ms, query); + + return result; + + /* Errors */ +no_decide_allocation: { + GST_WARNING_OBJECT(scope, "Subclass failed to decide allocation"); + gst_query_unref(query); + + return result; +} +} + +static gboolean +gst_pm_audio_visualizer_default_decide_allocation(GstPMAudioVisualizer *scope, + GstQuery *query) { + /* removed main memory pool implementation. This vmethod is overridden for + * using gl memory by gstglbaseaudiovisualizer. */ + g_error("vmethod gst_pm_audio_visualizer_default_decide_allocation is not " + "implemented"); +} + +GstFlowReturn +gst_pm_audio_visualizer_util_prepare_output_buffer(GstPMAudioVisualizer *scope, + GstBuffer **outbuf) { + GstPMAudioVisualizerPrivate *priv; + + priv = scope->priv; + + g_assert(priv->pool != NULL); + + /* we can't reuse the input buffer */ + if (!priv->pool_active) { + GST_DEBUG_OBJECT(scope, "setting pool %p active", priv->pool); + if (!gst_buffer_pool_set_active(priv->pool, TRUE)) + goto activate_failed; + priv->pool_active = TRUE; + } + GST_TRACE_OBJECT(scope, "using pool alloc"); + + return gst_buffer_pool_acquire_buffer(priv->pool, outbuf, NULL); + + /* ERRORS */ +activate_failed: { + GST_ELEMENT_ERROR(scope, RESOURCE, SETTINGS, + ("failed to activate bufferpool"), + ("failed to activate bufferpool")); + return GST_FLOW_ERROR; +} +} + +static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, + GstObject *parent, + GstBuffer *buffer) { + GstFlowReturn ret = GST_FLOW_OK; + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(parent); + GstPMAudioVisualizerClass *klass; + GstClockTime ts, running_time, frame_duration; + guint avail, sbpf; + // databuf is a buffer holding one video frame worth of audio data used as + // temp buffer for copying from the adapter only + // inbuf is a plugin-scoped buffer holding a copy of the one video frame worth + // of audio data from the adapter to process + GstBuffer *databuf, *inbuf; + gint bpf; + + klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); + + // ensure caps have been setup for sink and src pads, and plugin init code is + // done + g_mutex_lock(&scope->priv->config_lock); + while (!scope->priv->ready) { + g_cond_wait(&scope->priv->ready_cond, &scope->priv->config_lock); + } + g_mutex_unlock(&scope->priv->config_lock); + + if (buffer == NULL) { + return GST_FLOW_OK; + } + + /* remember pts timestamp of the first audio buffer as stream clock offset + * timestamp */ + g_mutex_lock(&scope->priv->config_lock); + if (!scope->priv->pts_offset_initialized) { + scope->priv->pts_offset_initialized = TRUE; + scope->priv->pts_offset = GST_BUFFER_PTS(buffer); + + GstClock *clock = gst_element_get_clock(GST_ELEMENT(scope)); + GstClockTime running_time = gst_clock_get_time(clock) - + gst_element_get_base_time(GST_ELEMENT(scope)); + + GST_DEBUG_OBJECT( + scope, + "Buffer ts: %" GST_TIME_FORMAT ", running_time: %" GST_TIME_FORMAT, + GST_TIME_ARGS(scope->priv->pts_offset), GST_TIME_ARGS(running_time)); + } + g_mutex_unlock(&scope->priv->config_lock); + + /* resync on DISCONT */ + if (GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DISCONT)) { + gst_adapter_clear(scope->priv->adapter); + } + + /* Make sure have an output format */ + if (gst_pad_check_reconfigure(scope->srcpad)) { + if (!gst_pm_audio_visualizer_src_negotiate(scope)) { + gst_pad_mark_reconfigure(scope->srcpad); + goto not_negotiated; + } + } + + bpf = GST_AUDIO_INFO_BPF(&scope->ainfo); + + if (bpf == 0) { + ret = GST_FLOW_NOT_NEGOTIATED; + goto beach; + } + + GST_TRACE_OBJECT(scope, "Chain func pushing %lu bytes to adapter", + gst_buffer_get_size(buffer)); + + gst_adapter_push(scope->priv->adapter, buffer); + + g_mutex_lock(&scope->priv->config_lock); + + /* this is what we want */ + /* number of audio bytes to process for one video frame */ + /* samples per video frame * audio bytes per frame for both channels */ + sbpf = scope->req_spf * bpf; + + inbuf = scope->priv->inbuf; + + /* original code FIXME: the timestamp in the adapter would be different - this + * should be fixed now by deriving timestamps from the number of samples + * consumed. */ + gst_buffer_copy_into(inbuf, buffer, GST_BUFFER_COPY_METADATA, 0, -1); + + /* this is what we have */ + avail = gst_adapter_available(scope->priv->adapter); + // GST_LOG_OBJECT(scope, "avail: %u, bpf: %u", avail, sbpf); + while (avail >= sbpf) { + + gboolean fps_changed_since_last_frame = scope->priv->fps_changed; + scope->priv->fps_changed = FALSE; + + // make sure frame duration does not change while processing one frame + frame_duration = scope->req_frame_duration; + + /* calculate timestamp based on audio input samples already processed to + * avoid clock drift */ + ts = scope->priv->pts_offset + + gst_util_uint64_scale_int(scope->priv->samples_consumed, GST_SECOND, + GST_AUDIO_INFO_RATE(&scope->ainfo)); + + scope->priv->samples_consumed += scope->req_spf; + + /* check for QoS, don't compute buffers that are known to be late */ + if (GST_CLOCK_TIME_IS_VALID(ts)) { + GstClockTime earliest_time; + gdouble proportion; + guint64 qostime; + + qostime = gst_segment_to_running_time(&scope->priv->segment, + GST_FORMAT_TIME, ts) + + frame_duration; + + earliest_time = scope->priv->earliest_time; + proportion = scope->priv->proportion; + + if (scope->priv->segment.format != GST_FORMAT_TIME) { + GST_WARNING_OBJECT(scope, + "Segment format not TIME, skipping QoS checks"); + } else if (GST_CLOCK_TIME_IS_VALID(earliest_time) && + qostime <= earliest_time) { + GstClockTime stream_time, jitter; + GstMessage *qos_msg; + + GST_DEBUG_OBJECT(scope, + "QoS: skip ts: %" GST_TIME_FORMAT + ", earliest: %" GST_TIME_FORMAT, + GST_TIME_ARGS(qostime), GST_TIME_ARGS(earliest_time)); + + ++scope->priv->dropped; + stream_time = gst_segment_to_stream_time(&scope->priv->segment, + GST_FORMAT_TIME, ts); + jitter = GST_CLOCK_DIFF(qostime, earliest_time); + qos_msg = + gst_message_new_qos(GST_OBJECT(scope), FALSE, qostime, stream_time, + ts, GST_BUFFER_DURATION(buffer)); + gst_message_set_qos_values(qos_msg, jitter, proportion, 1000000); + gst_message_set_qos_stats(qos_msg, GST_FORMAT_BUFFERS, + scope->priv->processed, scope->priv->dropped); + gst_element_post_message(GST_ELEMENT(scope), qos_msg); + + goto skip; + } + } + + /* map pts ts via segment for general use */ + ts = gst_segment_to_stream_time(&scope->priv->segment, GST_FORMAT_TIME, ts); + + /* get running time for passing through */ + running_time = + gst_segment_to_running_time(&scope->priv->segment, GST_FORMAT_TIME, ts); + + ++scope->priv->processed; + + /* sync controlled properties */ + if (GST_CLOCK_TIME_IS_VALID(ts)) + gst_object_sync_values(GST_OBJECT(scope), ts); + + /* this can fail as the data size we need could have changed */ + if (!(databuf = gst_adapter_get_buffer(scope->priv->adapter, sbpf))) + break; + + /* place sbpf number of bytes of audio data into inbuf */ + /* this is not a deep copy of the data at this point */ + gst_buffer_remove_all_memory(inbuf); + gst_buffer_copy_into(inbuf, databuf, GST_BUFFER_COPY_MEMORY, 0, sbpf); + gst_buffer_unref(databuf); + + /* call class->render() vmethod */ + g_mutex_unlock(&scope->priv->config_lock); + + ret = klass->render(scope, inbuf, ts, running_time, frame_duration); + if (ret != GST_FLOW_OK) { + goto beach; + } + + g_mutex_lock(&scope->priv->config_lock); + + skip: + // inform upstream of updated fps + if (fps_changed_since_last_frame == TRUE) { + gst_pm_audio_visualizer_send_latency_if_needed_unlocked(scope); + } + + /* we want to take less or more, depending on spf : req_spf */ + if (avail - sbpf >= sbpf) { + // enough audio data for more frames is available + gst_adapter_unmap(scope->priv->adapter); + gst_adapter_flush(scope->priv->adapter, sbpf); + } else if (avail >= sbpf) { + // was just enough audio data for one frame + /* just flush a bit and stop */ + // rendering. seems like a bug in the original code + // gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); + + // instead just flush one video frame worth of audio data from the buffer + // and stop + gst_adapter_unmap(scope->priv->adapter); + gst_adapter_flush(scope->priv->adapter, sbpf); + break; + } + avail = gst_adapter_available(scope->priv->adapter); + + // recalculate for the next frame + sbpf = scope->req_spf * bpf; + } + + g_mutex_unlock(&scope->priv->config_lock); + +beach: + return ret; + + /* ERRORS */ +not_negotiated: { + GST_DEBUG_OBJECT(scope, "Failed to renegotiate"); + return GST_FLOW_NOT_NEGOTIATED; +} +} + +static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, + GstObject *parent, + GstEvent *event) { + gboolean res; + GstPMAudioVisualizer *scope; + + scope = GST_PM_AUDIO_VISUALIZER(parent); + + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_QOS: { + gdouble proportion; + GstClockTimeDiff diff; + GstClockTime timestamp; + + gst_event_parse_qos(event, NULL, &proportion, &diff, ×tamp); + + /* save stuff for the _chain() function */ + g_mutex_lock(&scope->priv->config_lock); + // ignore QoS events for first few frames, sinks seem to send erratic QoS at + // the beginning + if (scope->priv->processed > QOS_IGNORE_FIRST_N_FRAMES) { + scope->priv->proportion = proportion; + if (diff > 0) { + /* we're late, this is a good estimate for next displayable + * frame (see part-qos.txt) */ + scope->priv->earliest_time = timestamp + MIN(diff * 2, GST_SECOND * 3) + + scope->req_frame_duration; + } else { + scope->priv->earliest_time = timestamp + diff; + } + } else { + GST_DEBUG_OBJECT(scope, "Ignoring early QoS event, processed frames: %d", + scope->priv->processed); + } + g_mutex_unlock(&scope->priv->config_lock); + + res = gst_pad_push_event(scope->priv->sinkpad, event); + break; + } + case GST_EVENT_LATENCY: + g_mutex_lock(&scope->priv->config_lock); + gst_event_parse_latency(event, &scope->latency); + g_mutex_unlock(&scope->priv->config_lock); + res = gst_pad_event_default(pad, parent, event); + GST_DEBUG_OBJECT(scope, "Received latency event: %" GST_TIME_FORMAT, + GST_TIME_ARGS(scope->latency)); + break; + case GST_EVENT_RECONFIGURE: + /* don't forward */ + gst_event_unref(event); + res = TRUE; + break; + default: + res = gst_pad_event_default(pad, parent, event); + break; + } + + return res; +} + +static gboolean gst_pm_audio_visualizer_sink_event(GstPad *pad, + GstObject *parent, + GstEvent *event) { + gboolean res; + + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(parent); + GstPMAudioVisualizerClass *klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); + + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_CAPS: { + GstCaps *caps; + + gst_event_parse_caps(event, &caps); + res = gst_pm_audio_visualizer_sink_setcaps(scope, caps); + gst_event_unref(event); + break; + } + case GST_EVENT_FLUSH_STOP: + g_mutex_lock(&scope->priv->config_lock); + gst_pm_audio_visualizer_reset_unlocked(scope); + g_mutex_unlock(&scope->priv->config_lock); + res = gst_pad_push_event(scope->srcpad, event); + break; + case GST_EVENT_SEGMENT: { + /* the newsegment values are used to clip the input samples + * and to convert the incoming timestamps to running time so + * we can do QoS */ + g_mutex_lock(&scope->priv->config_lock); + gst_event_copy_segment(event, &scope->priv->segment); + if (scope->priv->segment.format != GST_FORMAT_TIME) { + GST_WARNING_OBJECT(scope, "Unexpected segment format: %d", + scope->priv->segment.format); + } + scope->priv->pts_offset = + scope->priv->segment.start; // or segment.position if it's a live seek + scope->priv->pts_offset_initialized = TRUE; + scope->priv->samples_consumed = 0; + g_mutex_unlock(&scope->priv->config_lock); + if (klass->segment_change) { + klass->segment_change(scope, &scope->priv->segment); + } + res = gst_pad_push_event(scope->srcpad, event); + GST_DEBUG_OBJECT( + scope, "Segment start: %" GST_TIME_FORMAT ", stop: %" GST_TIME_FORMAT, + GST_TIME_ARGS(scope->priv->segment.start), + GST_TIME_ARGS(scope->priv->segment.stop)); + break; + } + default: + res = gst_pad_event_default(pad, parent, event); + break; + } + + return res; +} + +static GstClockTime calc_our_latency_unlocked(GstPMAudioVisualizer *scope, + gint rate) { + /* the max samples we must buffer */ + guint max_samples = MAX(scope->req_spf, scope->priv->spf); + return gst_util_uint64_scale(max_samples, GST_SECOND, rate); +} + +static void gst_pm_audio_visualizer_send_latency_if_needed_unlocked( + GstPMAudioVisualizer *scope) { + + // send latency event if latency changed a lot + GstClockTime latency = + calc_our_latency_unlocked(scope, GST_AUDIO_INFO_RATE(&scope->ainfo)); + + // check if the latency has changed enough to send an event + if (ABS((GstClockTimeDiff)latency - scope->priv->last_reported_latency) > + LATENCY_EVENT_MIN_CHANGE) { + + scope->priv->last_reported_latency = latency; + g_mutex_unlock(&scope->priv->config_lock); + gst_pad_push_event(scope->priv->sinkpad, gst_event_new_latency(latency)); + GST_INFO_OBJECT(scope, "Sent latency event to sink pad: %" GST_TIME_FORMAT, + GST_TIME_ARGS(latency)); + g_mutex_lock(&scope->priv->config_lock); + } +} + +static gboolean gst_pm_audio_visualizer_src_query(GstPad *pad, + GstObject *parent, + GstQuery *query) { + gboolean res = FALSE; + GstPMAudioVisualizer *scope; + + scope = GST_PM_AUDIO_VISUALIZER(parent); + + switch (GST_QUERY_TYPE(query)) { + case GST_QUERY_LATENCY: { + /* We need to send the query upstream and add the returned latency to our + * own */ + GstClockTime min_latency, max_latency; + gboolean us_live; + GstClockTime our_latency; + gint rate = GST_AUDIO_INFO_RATE(&scope->ainfo); + + if (rate == 0) + break; + + if ((res = gst_pad_peer_query(scope->priv->sinkpad, query))) { + gst_query_parse_latency(query, &us_live, &min_latency, &max_latency); + + GST_DEBUG_OBJECT( + scope, "Peer latency: min %" GST_TIME_FORMAT " max %" GST_TIME_FORMAT, + GST_TIME_ARGS(min_latency), GST_TIME_ARGS(max_latency)); + + g_mutex_lock(&scope->priv->config_lock); + our_latency = calc_our_latency_unlocked(scope, rate); + g_mutex_unlock(&scope->priv->config_lock); + + GST_DEBUG_OBJECT(scope, "Our latency: %" GST_TIME_FORMAT, + GST_TIME_ARGS(our_latency)); + + /* we add some latency but only if we need to buffer more than what + * upstream gives us */ + min_latency += our_latency; + if (max_latency != -1) + max_latency += our_latency; + + GST_DEBUG_OBJECT(scope, + "Calculated total latency : min %" GST_TIME_FORMAT + " max %" GST_TIME_FORMAT, + GST_TIME_ARGS(min_latency), GST_TIME_ARGS(max_latency)); + + gst_query_set_latency(query, TRUE, min_latency, max_latency); + g_mutex_lock(&scope->priv->config_lock); + scope->priv->last_reported_latency = our_latency; + g_mutex_unlock(&scope->priv->config_lock); + } + break; + } + default: + res = gst_pad_query_default(pad, parent, query); + break; + } + + return res; +} + +static GstStateChangeReturn +gst_pm_audio_visualizer_parent_change_state(GstElement *element, + GstStateChange transition) { + + GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + g_mutex_lock(&scope->priv->config_lock); + gst_pm_audio_visualizer_reset_unlocked(scope); + g_mutex_unlock(&scope->priv->config_lock); + break; + default: + break; + } + + GstStateChangeReturn ret = + GST_ELEMENT_CLASS(parent_class)->change_state(element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_pm_audio_visualizer_set_allocation(scope, NULL, NULL, NULL, NULL); + break; + case GST_STATE_CHANGE_READY_TO_NULL: + break; + default: + break; + } + + GstPMAudioVisualizerClass *klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); + return klass->change_state(element, transition); +} + +static GstStateChangeReturn +gst_pm_audio_visualizer_default_change_state(GstElement *element, + GstStateChange transition) { + return GST_STATE_CHANGE_SUCCESS; +} + +static gboolean log_fps_change(gpointer message) { + GST_INFO("%s", (gchar *)message); + + g_free(message); + return G_SOURCE_REMOVE; // remove after run +} + +void gst_pm_audio_visualizer_adjust_fps(GstPMAudioVisualizer *scope, + guint64 frame_duration) { + g_mutex_lock(&scope->priv->config_lock); + + guint64 set_duration; + guint set_req_spf; + + // clamp for cap fps + if (frame_duration <= scope->priv->caps_frame_duration) { + set_duration = scope->priv->caps_frame_duration; + set_req_spf = scope->priv->spf; + } else { + set_duration = frame_duration; + // calculate samples per frame for the given frame duration + set_req_spf = + (guint)(((guint64)GST_AUDIO_INFO_RATE(&scope->ainfo) * frame_duration + + GST_SECOND / 2) / + GST_SECOND); + } + + // update for next frame + if (scope->req_frame_duration != set_duration) { + scope->req_frame_duration = set_duration; + scope->req_spf = set_req_spf; + scope->priv->fps_changed = TRUE; + } + + g_mutex_unlock(&scope->priv->config_lock); + + if (gst_debug_category_get_threshold(pm_audio_visualizer_debug) >= + GST_LEVEL_WARNING) { + + gchar *message = + g_strdup_printf("Adjusting framerate, max fps: %f, using " + "frame-duration: %" GST_TIME_FORMAT ", spf: %u", + (gdouble)frame_duration / GST_SECOND, + GST_TIME_ARGS(set_duration), set_req_spf); + + g_idle_add(log_fps_change, message); + } +} diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h new file mode 100644 index 0000000..a96e526 --- /dev/null +++ b/src/gstpmaudiovisualizer.h @@ -0,0 +1,167 @@ +/* GStreamer + * Copyright (C) <2011> Stefan Kost + * Copyright (C) <2015> Luis de Bethencourt + * + * gstaudiovisualizer.c: base class for audio visualisation elements + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* + * The code in this file is based on + * GStreamer / gst-plugins-base, latest version as of 2025/05/29. + * gst-libs/gst/pbutils/gstaudiovisualizer.h Git Repository: + * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/blob/main/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.h + * + * Original copyright notice has been retained at the top of this file. + * The code has been modified to improve compatibility with projectM and OpenGL. + * See impl for details. + */ + +#ifndef __GST_PM_AUDIO_VISUALIZER_H__ +#define __GST_PM_AUDIO_VISUALIZER_H__ + +#include + +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_PM_AUDIO_VISUALIZER (gst_pm_audio_visualizer_get_type()) +#define GST_PM_AUDIO_VISUALIZER(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_PM_AUDIO_VISUALIZER, \ + GstPMAudioVisualizer)) +#define GST_PM_AUDIO_VISUALIZER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_PM_AUDIO_VISUALIZER, \ + GstPMAudioVisualizerClass)) +#define GST_PM_AUDIO_VISUALIZER_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_PM_AUDIO_VISUALIZER, \ + GstPMAudioVisualizerClass)) +#define GST_PM_IS_SYNAESTHESIA(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_PM_AUDIO_VISUALIZER)) +#define GST_PM_IS_SYNAESTHESIA_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_PM_AUDIO_VISUALIZER)) +typedef struct _GstPMAudioVisualizer GstPMAudioVisualizer; +typedef struct _GstPMAudioVisualizerClass GstPMAudioVisualizerClass; +typedef struct _GstPMAudioVisualizerPrivate GstPMAudioVisualizerPrivate; + +struct _GstPMAudioVisualizer { + GstElement parent; + + /** + * current min samples per frame wanted by the subclass (one channel), may + * vary depending on actual fps. + */ + guint req_spf; + + /** + * Current fps frame duration, may be different from caps fps. + */ + guint64 req_frame_duration; + + /** + * Caps video state. + */ + GstVideoInfo vinfo; + + /** + * Input audio state. + */ + GstAudioInfo ainfo; + + /*< private >*/ + GstPMAudioVisualizerPrivate *priv; + + /** + * Source pad to push video buffers downstream. + */ + GstPad *srcpad; + + /** + * Current pipeline latency. + */ + GstClockTime latency; +}; + +/** + * GstPMAudioVisualizerClass: + * @decide_allocation: buffer pool allocation + * @prepare_output_buffer: allocate a buffer for rendering a frame. + * @map_output_buffer: map video frame to memory buffer. + * @render: render a frame from an audio buffer. + * @setup: called whenever the format changes. + * + * Base class for audio visualizers, derived from gstreamer + * GstAudioVisualizerClass. This plugin handles rendering video frames with a + * fixed framerate from audio input samples. + */ +struct _GstPMAudioVisualizerClass { + /*< private >*/ + GstElementClass parent_class; + + /*< public >*/ + /** + * Virtual function, called whenever the caps change, sink and src pad caps + * are both configured. + */ + gboolean (*setup)(GstPMAudioVisualizer *scope); + + /** + * Virtual function for rendering a frame. + */ + GstFlowReturn (*render)(GstPMAudioVisualizer *scope, GstBuffer *audio, + GstClockTime pts, GstClockTime running_time, + guint64 frame_duration); + + /** + * Virtual function for buffer pool allocation. + */ + gboolean (*decide_allocation)(GstPMAudioVisualizer *scope, GstQuery *query); + + /** + * Virtual function to allow overridden change_state, cascading to GstElement. + */ + GstStateChangeReturn (*change_state)(GstElement *element, + GstStateChange transition); + + /** + * Virtual function to allow receiving segment change events. + */ + void (*segment_change)(GstPMAudioVisualizer *scope, GstSegment *segment); +}; + +GType gst_pm_audio_visualizer_get_type(void); + +/** + * Obtain buffer from buffer pool for rendering. + * + * @param scope Plugin data. + * @param outbuf Pointer for receiving output buffer. + * + * @return GST_FLOW_ERROR in case of pool errors, or the result of + * gst_buffer_pool_acquire_buffer(...) + */ +GstFlowReturn +gst_pm_audio_visualizer_util_prepare_output_buffer(GstPMAudioVisualizer *scope, + GstBuffer **outbuf); + +void gst_pm_audio_visualizer_adjust_fps(GstPMAudioVisualizer *scope, + guint64 frame_duration); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstPMAudioVisualizer, gst_object_unref) + +G_END_DECLS +#endif /* __GST_PM_AUDIO_VISUALIZER_H__ */ diff --git a/src/gstprojectm.c b/src/gstprojectm.c new file mode 100644 index 0000000..ff795c1 --- /dev/null +++ b/src/gstprojectm.c @@ -0,0 +1,182 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef USE_GLEW +#include +#endif + +#include "gstprojectm.h" + +#include +#include + +#include "debug.h" +#include "gstglbaseaudiovisualizer.h" +#include "gstprojectmcaps.h" + +GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); +#define GST_CAT_DEFAULT gst_projectm_debug + +struct _GstProjectMPrivate { + + GstBaseProjectMPrivate base; +}; + +G_DEFINE_TYPE_WITH_CODE(GstProjectM, gst_projectm, + GST_TYPE_GL_BASE_AUDIO_VISUALIZER, + G_ADD_PRIVATE(GstProjectM) + GST_DEBUG_CATEGORY_INIT(gst_projectm_debug, + "gstprojectm", 0, + "Plugin Root")); + +void gst_projectm_set_property(GObject *object, guint property_id, + const GValue *value, GParamSpec *pspec) { + + GstProjectM *plugin = GST_PROJECTM(object); + + gst_projectm_base_set_property(object, &plugin->settings, property_id, value, + pspec); +} + +void gst_projectm_get_property(GObject *object, guint property_id, + GValue *value, GParamSpec *pspec) { + GstProjectM *plugin = GST_PROJECTM(object); + + gst_projectm_base_get_property(object, &plugin->settings, property_id, value, + pspec); +} + +static void gst_projectm_init(GstProjectM *plugin) { + plugin->priv = gst_projectm_get_instance_private(plugin); + + gst_gl_memory_init_once(); + + gst_projectm_base_init(&plugin->settings, &plugin->priv->base); +} + +static void gst_projectm_finalize(GObject *object) { + + GstProjectM *plugin = GST_PROJECTM(object); + + gst_projectm_base_finalize(&plugin->settings, &plugin->priv->base); + G_OBJECT_CLASS(gst_projectm_parent_class)->finalize(object); +} + +static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { + + GstProjectM *plugin = GST_PROJECTM(src); + + gst_projectm_base_gl_stop(G_OBJECT(src), &plugin->priv->base); +} + +static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { + // Cast the audio visualizer to the ProjectM plugin + GstProjectM *plugin = GST_PROJECTM(glav); + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(glav); + + gst_projectm_base_gl_start(G_OBJECT(glav), &plugin->priv->base, + &plugin->settings, glav->context, &pmav->vinfo); + + GST_INFO_OBJECT(plugin, "GL start complete"); + + return TRUE; +} + +static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { + + GstPMAudioVisualizer *pmav = GST_PM_AUDIO_VISUALIZER(glav); + + // Log audio info + GST_DEBUG_OBJECT( + glav, "Audio Information ", + pmav->ainfo.channels, pmav->ainfo.rate, pmav->ainfo.finfo->description); + + // Log video info + GST_DEBUG_OBJECT( + glav, + "Video Information ", + GST_VIDEO_INFO_WIDTH(&pmav->vinfo), GST_VIDEO_INFO_HEIGHT(&pmav->vinfo), + pmav->vinfo.fps_n, pmav->vinfo.fps_d, pmav->req_spf); + + return TRUE; +} + +static gboolean gst_projectm_fill_gl_memory_callback(gpointer stuff) { + + GstAVRenderParams *render_data = (GstAVRenderParams *)stuff; + GstProjectM *plugin = GST_PROJECTM(render_data->plugin); + GstGLBaseAudioVisualizer *glav = + GST_GL_BASE_AUDIO_VISUALIZER(render_data->plugin); + gboolean result = TRUE; + + // VIDEO + GST_TRACE_OBJECT(plugin, "rendering projectM to fbo %d", + render_data->fbo->fbo_id); + + gst_projectm_base_fill_gl_memory_callback(&plugin->priv->base, glav->context, + render_data->fbo, render_data->pts, + render_data->in_audio); + + return result; +} + +static gboolean gst_projectm_fill_gl_memory(GstAVRenderParams *render_data) { + + gboolean result = gst_gl_framebuffer_draw_to_texture( + render_data->fbo, render_data->mem, gst_projectm_fill_gl_memory_callback, + render_data); + + return result; +} + +static void gst_projectm_segment_change(GstPMAudioVisualizer *scope, + GstSegment *segment) { + GstProjectM *plugin = GST_PROJECTM(scope); + gint64 pts_offset = segment->time - segment->start; + gst_projectm_base_set_segment_pts_offset(&plugin->priv->base, pts_offset); +} + +static void gst_projectm_class_init(GstProjectMClass *klass) { + GObjectClass *gobject_class = (GObjectClass *)klass; + GstPMAudioVisualizerClass *parent_scope_class = + GST_PM_AUDIO_VISUALIZER_CLASS(klass); + GstGLBaseAudioVisualizerClass *scope_class = + GST_GL_BASE_AUDIO_VISUALIZER_CLASS(klass); + + // Setup audio and video caps + const gchar *audio_sink_caps = get_audio_sink_cap(); + const gchar *video_src_caps = get_video_src_cap(); + + gst_element_class_add_pad_template( + GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + gst_caps_from_string(video_src_caps))); + gst_element_class_add_pad_template( + GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + gst_caps_from_string(audio_sink_caps))); + + gst_element_class_set_static_metadata( + GST_ELEMENT_CLASS(klass), "ProjectM Visualizer", "Generic", + "A plugin for visualizing music using ProjectM", + "AnomieVision | Tristan Charpentier " + " | Michael Baetgen " + ""); + + // Setup properties + gobject_class->set_property = gst_projectm_set_property; + gobject_class->get_property = gst_projectm_get_property; + + gst_projectm_base_install_properties(gobject_class); + + gobject_class->finalize = gst_projectm_finalize; + + scope_class->supported_gl_api = GST_GL_API_OPENGL3 | GST_GL_API_GLES2; + scope_class->gl_start = GST_DEBUG_FUNCPTR(gst_projectm_gl_start); + scope_class->gl_stop = GST_DEBUG_FUNCPTR(gst_projectm_gl_stop); + scope_class->fill_gl_memory = GST_DEBUG_FUNCPTR(gst_projectm_fill_gl_memory); + scope_class->setup = GST_DEBUG_FUNCPTR(gst_projectm_setup); + parent_scope_class->segment_change = + GST_DEBUG_FUNCPTR(gst_projectm_segment_change); +} diff --git a/src/plugin.h b/src/gstprojectm.h similarity index 66% rename from src/plugin.h rename to src/gstprojectm.h index de1acff..aa2a001 100644 --- a/src/plugin.h +++ b/src/gstprojectm.h @@ -2,7 +2,7 @@ #define __GST_PROJECTM_H__ #include "gstglbaseaudiovisualizer.h" -#include +#include "gstprojectmbase.h" typedef struct _GstProjectMPrivate GstProjectMPrivate; @@ -12,31 +12,23 @@ G_BEGIN_DECLS G_DECLARE_FINAL_TYPE(GstProjectM, gst_projectm, GST, PROJECTM, GstGLBaseAudioVisualizer) +/* + * Main plug-in. Handles interactions with projectM. + * Uses GstPMAudioVisualizer for handling audio-visualization (audio input, + * timing, video frame data). GstGLBaseAudioVisualizer extends + * GstPMAudioVisualizer to add gl context handling and is used by this plugin + * directly. GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. + */ struct _GstProjectM { GstGLBaseAudioVisualizer element; - gchar *preset_path; - gchar *texture_dir_path; - - gfloat beat_sensitivity; - gdouble hard_cut_duration; - gboolean hard_cut_enabled; - gfloat hard_cut_sensitivity; - gdouble soft_cut_duration; - gdouble preset_duration; - gulong mesh_width; - gulong mesh_height; - gboolean aspect_correction; - gfloat easter_egg; - gboolean preset_locked; - gboolean enable_playlist; - gboolean shuffle_presets; + GstBaseProjectMSettings settings; GstProjectMPrivate *priv; }; struct _GstProjectMClass { - GstAudioVisualizerClass parent_class; + GstGLBaseAudioVisualizerClass parent_class; }; static void gst_projectm_set_property(GObject *object, guint prop_id, @@ -53,8 +45,7 @@ static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav); static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *glav); -static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, - GstBuffer *audio, GstVideoFrame *video); +static gboolean gst_projectm_fill_gl_memory(GstAVRenderParams *render_data); static void gst_projectm_class_init(GstProjectMClass *klass); diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c new file mode 100644 index 0000000..6c9fad2 --- /dev/null +++ b/src/gstprojectmbase.c @@ -0,0 +1,690 @@ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstprojectmbase.h" + +#include "gstprojectmconfig.h" + +#include "debug.h" +#include "gstglbaseaudiovisualizer.h" + +#include + +GST_DEBUG_CATEGORY_STATIC(gst_projectm_base_debug); +#define GST_CAT_DEFAULT gst_projectm_base_debug + +enum { + PROP_0, + PROP_PRESET_PATH, + PROP_TEXTURE_DIR_PATH, + PROP_BEAT_SENSITIVITY, + PROP_HARD_CUT_DURATION, + PROP_HARD_CUT_ENABLED, + PROP_HARD_CUT_SENSITIVITY, + PROP_SOFT_CUT_DURATION, + PROP_PRESET_DURATION, + PROP_MESH_SIZE, + PROP_ASPECT_CORRECTION, + PROP_EASTER_EGG, + PROP_PRESET_LOCKED, + PROP_SHUFFLE_PRESETS, + PROP_ENABLE_PLAYLIST, + PROP_MIN_FPS +}; + +/** + * @brief ProjectM Settings (defaults) + */ + +#define DEFAULT_PRESET_PATH NULL +#define DEFAULT_TEXTURE_DIR_PATH NULL +#define DEFAULT_BEAT_SENSITIVITY 1.0 +#define DEFAULT_HARD_CUT_DURATION 3.0 +#define DEFAULT_HARD_CUT_ENABLED FALSE +#define DEFAULT_HARD_CUT_SENSITIVITY 1.0 +#define DEFAULT_SOFT_CUT_DURATION 3.0 +#define DEFAULT_PRESET_DURATION 0.0 +#define DEFAULT_MESH_SIZE "48,32" +#define DEFAULT_ASPECT_CORRECTION TRUE +#define DEFAULT_EASTER_EGG 0.0 +#define DEFAULT_PRESET_LOCKED FALSE +#define DEFAULT_ENABLE_PLAYLIST TRUE +#define DEFAULT_SHUFFLE_PRESETS TRUE // depends on ENABLE_PLAYLIST +#define DEFAULT_MIN_FPS "1/1" + +void gst_projectm_base_init_once() { + GST_DEBUG_CATEGORY_INIT(gst_projectm_base_debug, "projectm_base", 0, + "projectM visualizer plugin base"); +} + +static gboolean gst_projectm_base_log_preset_change(gpointer preset) { + GST_INFO("Preset: %s", (char *)preset); + + projectm_free_string((char *)preset); + + return G_SOURCE_REMOVE; // remove after run +} + +gboolean gst_projectm_base_parse_fraction(const gchar *str, gint *numerator, + gint *denominator) { + g_return_val_if_fail(str != NULL, FALSE); + g_return_val_if_fail(numerator != NULL, FALSE); + g_return_val_if_fail(denominator != NULL, FALSE); + + gchar **parts = g_strsplit(str, "/", 2); + if (!parts[0] || !parts[1]) { + g_strfreev(parts); + return FALSE; + } + + gchar *endptr = NULL; + gint64 num = g_ascii_strtoll(parts[0], &endptr, 10); + if (*endptr != '\0') { + g_strfreev(parts); + return FALSE; + } + + gint64 denom = g_ascii_strtoll(parts[1], &endptr, 10); + if (*endptr != '\0' || denom == 0) { + g_strfreev(parts); + return FALSE; + } + + *numerator = (gint)num; + *denominator = (gint)denom; + + g_strfreev(parts); + return TRUE; +} + +static void gst_projectm_base_handle_preset_change(bool is_hard_cut, + unsigned int index, + void *user_data) { + + if (gst_debug_category_get_threshold(gst_projectm_base_debug) >= + GST_LEVEL_INFO) { + char *name = + projectm_playlist_item((projectm_playlist_handle)user_data, index); + + g_idle_add(gst_projectm_base_log_preset_change, name); + } +} + +static GstBaseProjectMInitResult +projectm_init(GObject *plugin, GstBaseProjectMSettings *settings, + GstVideoInfo *vinfo) { + + GstBaseProjectMInitResult result; + result.ret_handle = NULL; + result.ret_playlist = NULL; + result.success = FALSE; + + // Create ProjectM instance + GST_DEBUG_OBJECT(plugin, "Creating projectM instance.."); + result.ret_handle = projectm_create(); + + if (!result.ret_handle) { + GST_DEBUG_OBJECT( + plugin, + "project_create() returned NULL, projectM instance was not created!"); + + return result; + } else { + GST_DEBUG_OBJECT(plugin, "Created projectM instance!"); + } + + if (settings->enable_playlist) { + GST_DEBUG_OBJECT(plugin, "Playlist enabled"); + + // initialize preset playlist + result.ret_playlist = projectm_playlist_create(result.ret_handle); + projectm_playlist_set_shuffle(result.ret_playlist, + settings->shuffle_presets); + + // add handler to print preset change + projectm_playlist_set_preset_switched_event_callback( + result.ret_playlist, gst_projectm_base_handle_preset_change, + result.ret_playlist); + } else { + GST_DEBUG_OBJECT(plugin, "Playlist disabled"); + } + // Log properties + GST_INFO_OBJECT( + plugin, + "Using Properties: " + "preset=%s, " + "texture-dir=%s, " + "beat-sensitivity=%f, " + "hard-cut-duration=%f, " + "hard-cut-enabled=%d, " + "hard-cut-sensitivity=%f, " + "soft-cut-duration=%f, " + "preset-duration=%f, " + "mesh-size=(%lu, %lu)" + "aspect-correction=%d, " + "easter-egg=%f, " + "preset-locked=%d, " + "enable-playlist=%d, " + "shuffle-presets=%d, " + "min-fps=%d/%d", + settings->preset_path, settings->texture_dir_path, + settings->beat_sensitivity, settings->hard_cut_duration, + settings->hard_cut_enabled, settings->hard_cut_sensitivity, + settings->soft_cut_duration, settings->preset_duration, + settings->mesh_width, settings->mesh_height, settings->aspect_correction, + settings->easter_egg, settings->preset_locked, settings->enable_playlist, + settings->shuffle_presets, settings->min_fps_n, settings->min_fps_d); + + // Load preset file if path is provided + if (settings->preset_path != NULL) { + if (result.ret_playlist != NULL) { + unsigned int added_count = projectm_playlist_add_path( + result.ret_playlist, settings->preset_path, true, false); + GST_INFO_OBJECT(plugin, "Loaded preset path: %s, presets found: %d", + settings->preset_path, added_count); + } else { + projectm_load_preset_file(result.ret_handle, settings->preset_path, + false); + GST_INFO_OBJECT(plugin, "Loaded preset file: %s", settings->preset_path); + } + } + + // Set texture search path if directory path is provided + if (settings->texture_dir_path != NULL) { + const gchar *texturePaths[1] = {settings->texture_dir_path}; + projectm_set_texture_search_paths(result.ret_handle, texturePaths, 1); + } + + // Set properties + projectm_set_beat_sensitivity(result.ret_handle, settings->beat_sensitivity); + projectm_set_hard_cut_duration(result.ret_handle, + settings->hard_cut_duration); + projectm_set_hard_cut_enabled(result.ret_handle, settings->hard_cut_enabled); + projectm_set_hard_cut_sensitivity(result.ret_handle, + settings->hard_cut_sensitivity); + projectm_set_soft_cut_duration(result.ret_handle, + settings->soft_cut_duration); + + // Set preset duration, or set to in infinite duration if zero + if (settings->preset_duration > 0.0) { + projectm_set_preset_duration(result.ret_handle, settings->preset_duration); + // kick off the first preset + if (projectm_playlist_size(result.ret_playlist) > 1 && + !settings->preset_locked) { + projectm_playlist_play_next(result.ret_playlist, true); + } + } else { + projectm_set_preset_duration(result.ret_handle, 999999.0); + } + + projectm_set_mesh_size(result.ret_handle, settings->mesh_width, + settings->mesh_height); + projectm_set_aspect_correction(result.ret_handle, + settings->aspect_correction); + projectm_set_easter_egg(result.ret_handle, settings->easter_egg); + projectm_set_preset_locked(result.ret_handle, settings->preset_locked); + + gdouble fps; + gst_util_fraction_to_double(GST_VIDEO_INFO_FPS_N(vinfo), + GST_VIDEO_INFO_FPS_D(vinfo), &fps); + + projectm_set_fps(result.ret_handle, gst_util_gdouble_to_guint64(fps)); + projectm_set_window_size(result.ret_handle, GST_VIDEO_INFO_WIDTH(vinfo), + GST_VIDEO_INFO_HEIGHT(vinfo)); + + result.success = TRUE; + return result; +} + +void gst_projectm_base_set_property(GObject *object, + GstBaseProjectMSettings *settings, + guint property_id, const GValue *value, + GParamSpec *pspec) { + + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); + + const gchar *property_name = g_param_spec_get_name(pspec); + GST_DEBUG_OBJECT(object, "set-property <%s>", property_name); + + switch (property_id) { + case PROP_PRESET_PATH: + settings->preset_path = g_strdup(g_value_get_string(value)); + break; + case PROP_TEXTURE_DIR_PATH: + settings->texture_dir_path = g_strdup(g_value_get_string(value)); + break; + case PROP_BEAT_SENSITIVITY: + settings->beat_sensitivity = g_value_get_float(value); + break; + case PROP_HARD_CUT_DURATION: + settings->hard_cut_duration = g_value_get_double(value); + break; + case PROP_HARD_CUT_ENABLED: + settings->hard_cut_enabled = g_value_get_boolean(value); + break; + case PROP_HARD_CUT_SENSITIVITY: + settings->hard_cut_sensitivity = g_value_get_float(value); + break; + case PROP_SOFT_CUT_DURATION: + settings->soft_cut_duration = g_value_get_double(value); + break; + case PROP_PRESET_DURATION: + settings->preset_duration = g_value_get_double(value); + break; + case PROP_MESH_SIZE: { + const gchar *meshSizeStr = g_value_get_string(value); + gint width, height; + + gchar **parts = g_strsplit(meshSizeStr, ",", 2); + + if (parts && g_strv_length(parts) == 2) { + width = atoi(parts[0]); + height = atoi(parts[1]); + + settings->mesh_width = width; + settings->mesh_height = height; + + g_strfreev(parts); + } + } break; + case PROP_ASPECT_CORRECTION: + settings->aspect_correction = g_value_get_boolean(value); + break; + case PROP_EASTER_EGG: + settings->easter_egg = g_value_get_float(value); + break; + case PROP_PRESET_LOCKED: + settings->preset_locked = g_value_get_boolean(value); + break; + case PROP_ENABLE_PLAYLIST: + settings->enable_playlist = g_value_get_boolean(value); + break; + case PROP_SHUFFLE_PRESETS: + settings->shuffle_presets = g_value_get_boolean(value); + break; + case PROP_MIN_FPS: + gint num, denom; + gboolean success; + const gchar *fpsStr = g_value_get_string(value); + success = gst_projectm_base_parse_fraction(fpsStr, &num, &denom); + if (success) { + settings->min_fps_n = num; + settings->min_fps_d = denom; + g_object_set(G_OBJECT(glav), "min-fps-n", num, "min-fps-d", denom, NULL); + } + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); + break; + } +} + +void gst_projectm_base_get_property(GObject *object, + GstBaseProjectMSettings *settings, + guint property_id, GValue *value, + GParamSpec *pspec) { + + const gchar *property_name = g_param_spec_get_name(pspec); + GST_DEBUG_OBJECT(settings, "get-property <%s>", property_name); + + GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); + + switch (property_id) { + case PROP_PRESET_PATH: + g_value_set_string(value, settings->preset_path); + break; + case PROP_TEXTURE_DIR_PATH: + g_value_set_string(value, settings->texture_dir_path); + break; + case PROP_BEAT_SENSITIVITY: + g_value_set_float(value, settings->beat_sensitivity); + break; + case PROP_HARD_CUT_DURATION: + g_value_set_double(value, settings->hard_cut_duration); + break; + case PROP_HARD_CUT_ENABLED: + g_value_set_boolean(value, settings->hard_cut_enabled); + break; + case PROP_HARD_CUT_SENSITIVITY: + g_value_set_float(value, settings->hard_cut_sensitivity); + break; + case PROP_SOFT_CUT_DURATION: + g_value_set_double(value, settings->soft_cut_duration); + break; + case PROP_PRESET_DURATION: + g_value_set_double(value, settings->preset_duration); + break; + case PROP_MESH_SIZE: { + gchar *meshSizeStr = + g_strdup_printf("%lu,%lu", settings->mesh_width, settings->mesh_height); + g_value_set_string(value, meshSizeStr); + g_free(meshSizeStr); + break; + } + case PROP_ASPECT_CORRECTION: + g_value_set_boolean(value, settings->aspect_correction); + break; + case PROP_EASTER_EGG: + g_value_set_float(value, settings->easter_egg); + break; + case PROP_PRESET_LOCKED: + g_value_set_boolean(value, settings->preset_locked); + break; + case PROP_ENABLE_PLAYLIST: + g_value_set_boolean(value, settings->enable_playlist); + break; + case PROP_SHUFFLE_PRESETS: + g_value_set_boolean(value, settings->shuffle_presets); + break; + case PROP_MIN_FPS: + gchar *fpsStr = + g_strdup_printf("%d/%d", settings->min_fps_n, settings->min_fps_d); + g_value_set_string(value, fpsStr); + g_free(fpsStr); + + g_object_set(G_OBJECT(glav), "min-fps-n", settings->min_fps_n, "min-fps-d", + settings->min_fps_d, NULL); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); + break; + } +} + +void gst_projectm_base_init(GstBaseProjectMSettings *settings, + GstBaseProjectMPrivate *priv) { + + // Set default values for properties + settings->preset_path = DEFAULT_PRESET_PATH; + settings->texture_dir_path = DEFAULT_TEXTURE_DIR_PATH; + settings->beat_sensitivity = DEFAULT_BEAT_SENSITIVITY; + settings->hard_cut_duration = DEFAULT_HARD_CUT_DURATION; + settings->hard_cut_enabled = DEFAULT_HARD_CUT_ENABLED; + settings->hard_cut_sensitivity = DEFAULT_HARD_CUT_SENSITIVITY; + settings->soft_cut_duration = DEFAULT_SOFT_CUT_DURATION; + settings->preset_duration = DEFAULT_PRESET_DURATION; + settings->enable_playlist = DEFAULT_ENABLE_PLAYLIST; + settings->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; + + const gchar *meshSizeStr = DEFAULT_MESH_SIZE; + gint width, height; + + gchar **parts = g_strsplit(meshSizeStr, ",", 2); + + if (parts && g_strv_length(parts) == 2) { + width = atoi(parts[0]); + height = atoi(parts[1]); + + settings->mesh_width = width; + settings->mesh_height = height; + + g_strfreev(parts); + } + + settings->aspect_correction = DEFAULT_ASPECT_CORRECTION; + settings->easter_egg = DEFAULT_EASTER_EGG; + settings->preset_locked = DEFAULT_PRESET_LOCKED; + + gst_projectm_base_parse_fraction(DEFAULT_MIN_FPS, &settings->min_fps_n, + &settings->min_fps_d); + + priv->first_frame_time = 0; + priv->first_frame_received = FALSE; + + g_mutex_init(&priv->projectm_lock); +} + +void gst_projectm_base_finalize(GstBaseProjectMSettings *settings, + GstBaseProjectMPrivate *priv) { + g_free(settings->preset_path); + g_free(settings->texture_dir_path); + g_mutex_clear(&priv->projectm_lock); +} + +gboolean gst_projectm_base_gl_start(GObject *plugin, + GstBaseProjectMPrivate *priv, + GstBaseProjectMSettings *settings, + GstGLContext *context, + GstVideoInfo *vinfo) { + +#ifdef USE_GLEW + GST_DEBUG_OBJECT(plugin, "Initializing GLEW"); + GLenum err = glewInit(); + if (GLEW_OK != err) { + GST_ERROR_OBJECT(plugin, "GLEW initialization failed"); + return FALSE; + } +#endif + + GST_PROJECTM_BASE_LOCK(priv); + + // Check if ProjectM instance exists, and create if not + if (!priv->handle) { + // Create ProjectM instance + priv->first_frame_received = FALSE; + GstBaseProjectMInitResult result = projectm_init(plugin, settings, vinfo); + if (!result.success) { + GST_ERROR_OBJECT(plugin, "projectM could not be initialized"); + return FALSE; + } + gl_error_handler(context); + priv->handle = result.ret_handle; + priv->playlist = result.ret_playlist; + } + GST_PROJECTM_BASE_UNLOCK(priv); + + GST_INFO_OBJECT(plugin, "projectM GL start complete"); + return TRUE; +} + +void gst_projectm_base_gl_stop(GObject *plugin, GstBaseProjectMPrivate *priv) { + + GST_PROJECTM_BASE_LOCK(priv); + if (priv->handle) { + GST_DEBUG_OBJECT(plugin, "Destroying ProjectM instance"); + projectm_destroy(priv->handle); + priv->handle = NULL; + } + GST_PROJECTM_BASE_UNLOCK(priv); +} + +gdouble get_seconds_since_first_frame_unlocked(GstBaseProjectMPrivate *priv, + GstClockTime pts) { + if (!priv->first_frame_received) { + // store the timestamp of the first frame + priv->first_frame_time = pts; + priv->first_frame_received = TRUE; + return 0.0; + } + + // calculate elapsed time + GstClockTime elapsed_time = pts - priv->first_frame_time; + + // convert to fractional seconds + gdouble elapsed_seconds = (gdouble)elapsed_time / GST_SECOND; + + return elapsed_seconds; +} + +void gst_projectm_base_fill_audio_buffer_unlocked(GstBaseProjectMPrivate *priv, + GstBuffer *in_audio) { + + if (in_audio != NULL) { + + GstMapInfo audioMap; + + gst_buffer_map(in_audio, &audioMap, GST_MAP_READ); + + projectm_pcm_add_int16(priv->handle, (gint16 *)audioMap.data, + audioMap.size / 4, PROJECTM_STEREO); + + gst_buffer_unmap(in_audio, &audioMap); + } +} + +void gst_projectm_base_fill_gl_memory_callback(GstBaseProjectMPrivate *priv, + GstGLContext *context, + GstGLFramebuffer *fbo, + GstClockTime pts, + GstBuffer *in_audio) { + + GST_PROJECTM_BASE_LOCK(priv); + + // get current gst sync time (pts) and set projectM time + gdouble seconds_since_first_frame = + get_seconds_since_first_frame_unlocked(priv, pts); + + projectm_set_frame_time(priv->handle, seconds_since_first_frame); + + // process audio buffer + gst_projectm_base_fill_audio_buffer_unlocked(priv, in_audio); + + // render the frame + projectm_opengl_render_frame_fbo(priv->handle, fbo->fbo_id); + + // removed for performance reasons: gl_error_handler(context); + + GST_PROJECTM_BASE_UNLOCK(priv); +} + +void gst_projectm_base_set_segment_pts_offset(GstBaseProjectMPrivate *priv, + gint64 pts_offset) { + GST_PROJECTM_BASE_LOCK(priv); + priv->first_frame_time = pts_offset; + GST_PROJECTM_BASE_UNLOCK(priv); +} + +void gst_projectm_base_install_properties(GObjectClass *gobject_class) { + + // setup properties + g_object_class_install_property( + gobject_class, PROP_PRESET_PATH, + g_param_spec_string( + "preset", "Preset", + "Specifies the path to the preset file. The preset file determines " + "the visual style and behavior of the audio visualizer.", + DEFAULT_PRESET_PATH, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_TEXTURE_DIR_PATH, + g_param_spec_string("texture-dir", "Texture Directory", + "Sets the path to the directory containing textures " + "used in the visualizer.", + DEFAULT_TEXTURE_DIR_PATH, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_BEAT_SENSITIVITY, + g_param_spec_float( + "beat-sensitivity", "Beat Sensitivity", + "Controls the sensitivity to audio beats. Higher values make the " + "visualizer respond more strongly to beats.", + 0.0, 5.0, DEFAULT_BEAT_SENSITIVITY, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_HARD_CUT_DURATION, + g_param_spec_double("hard-cut-duration", "Hard Cut Duration", + "Sets the duration, in seconds, for hard cuts. Hard " + "cuts are abrupt transitions in the visualizer.", + 0.0, 999999.0, DEFAULT_HARD_CUT_DURATION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_HARD_CUT_ENABLED, + g_param_spec_boolean( + "hard-cut-enabled", "Hard Cut Enabled", + "Enables or disables hard cuts. When enabled, the visualizer may " + "exhibit sudden transitions based on the audio input.", + DEFAULT_HARD_CUT_ENABLED, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_HARD_CUT_SENSITIVITY, + g_param_spec_float( + "hard-cut-sensitivity", "Hard Cut Sensitivity", + "Adjusts the sensitivity of the visualizer to hard cuts. Higher " + "values increase the responsiveness to abrupt changes in audio.", + 0.0, 1.0, DEFAULT_HARD_CUT_SENSITIVITY, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_SOFT_CUT_DURATION, + g_param_spec_double( + "soft-cut-duration", "Soft Cut Duration", + "Sets the duration, in seconds, for soft cuts. Soft cuts are " + "smoother transitions between visualizer states.", + 0.0, 999999.0, DEFAULT_SOFT_CUT_DURATION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_PRESET_DURATION, + g_param_spec_double("preset-duration", "Preset Duration", + "Sets the duration, in seconds, for each preset. A " + "zero value causes the preset to play indefinitely.", + 0.0, 999999.0, DEFAULT_PRESET_DURATION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_MESH_SIZE, + g_param_spec_string("mesh-size", "Mesh Size", + "Sets the size of the mesh used in rendering. The " + "format is 'width,height'.", + DEFAULT_MESH_SIZE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_ASPECT_CORRECTION, + g_param_spec_boolean( + "aspect-correction", "Aspect Correction", + "Enables or disables aspect ratio correction. When enabled, the " + "visualizer adjusts for aspect ratio differences in rendering.", + DEFAULT_ASPECT_CORRECTION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_EASTER_EGG, + g_param_spec_float( + "easter-egg", "Easter Egg", + "Controls the activation of an Easter Egg feature. The value " + "determines the likelihood of triggering the Easter Egg.", + 0.0, 1.0, DEFAULT_EASTER_EGG, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_PRESET_LOCKED, + g_param_spec_boolean( + "preset-locked", "Preset Locked", + "Locks or unlocks the current preset. When locked, the visualizer " + "remains on the current preset without automatic changes.", + DEFAULT_PRESET_LOCKED, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_ENABLE_PLAYLIST, + g_param_spec_boolean( + "enable-playlist", "Enable Playlist", + "Enables or disables the playlist feature. When enabled, the " + "visualizer can switch between presets based on a provided playlist.", + DEFAULT_ENABLE_PLAYLIST, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_SHUFFLE_PRESETS, + g_param_spec_boolean( + "shuffle-presets", "Shuffle Presets", + "Enables or disables preset shuffling. When enabled, the visualizer " + "randomly selects presets from the playlist if presets are provided " + "and not locked. Playlist must be enabled for this to take effect.", + DEFAULT_SHUFFLE_PRESETS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_MIN_FPS, + g_param_spec_string( + "min-fps", "Minimum FPS", + "Specifies the lower bound for EMA fps adjustments for real-time " + "pipelines. How low the fps is allowed to be in case the rendering " + "can't keep up with pipeline fps. Applies to real-time pipelines " + "only.", + DEFAULT_MIN_FPS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); +} diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h new file mode 100644 index 0000000..952e571 --- /dev/null +++ b/src/gstprojectmbase.h @@ -0,0 +1,200 @@ + +/* + * Basic gst/projectM integration structs and functions that can be re-used for + * alternative plugin implementations. + */ + +#ifndef __GST_PROJECTM_BASE_H__ +#define __GST_PROJECTM_BASE_H__ + +#include +#include +#include +#include + +G_BEGIN_DECLS + +/** + * projectM config properties. + */ +struct _GstBaseProjectMSettings { + + gchar *preset_path; + gchar *texture_dir_path; + + gfloat beat_sensitivity; + gdouble hard_cut_duration; + gboolean hard_cut_enabled; + gfloat hard_cut_sensitivity; + gdouble soft_cut_duration; + gdouble preset_duration; + gulong mesh_width; + gulong mesh_height; + gboolean aspect_correction; + gfloat easter_egg; + gboolean preset_locked; + gboolean enable_playlist; + gboolean shuffle_presets; + gint min_fps_n; + gint min_fps_d; +}; + +/** + * Variables needed for managing projectM. + */ +struct _GstBaseProjectMPrivate { + projectm_handle handle; + projectm_playlist_handle playlist; + GMutex projectm_lock; + + GstClockTime first_frame_time; + gboolean first_frame_received; +}; + +/** + * projectM init result return arguments. + */ +struct _GstBaseProjectMInitResult { + projectm_handle ret_handle; + projectm_playlist_handle ret_playlist; + gboolean success; +}; + +typedef struct _GstBaseProjectMPrivate GstBaseProjectMPrivate; +typedef struct _GstBaseProjectMSettings GstBaseProjectMSettings; +typedef struct _GstBaseProjectMInitResult GstBaseProjectMInitResult; + +/** + * One time initialization. Should be called once before any other function in + * this unit. + */ +void gst_projectm_base_init_once(); + +/** + * get_property delegate for projectM setting structs. + * + * @param object Plugin gst object. + * @param settings Settings struct to update. + * @param property_id Property id to update. + * @param value Property value. + * @param pspec Gst param type spec. + */ +void gst_projectm_base_set_property(GObject *object, + GstBaseProjectMSettings *settings, + guint property_id, const GValue *value, + GParamSpec *pspec); + +/** + * set_property delegate for projectM setting structs. + * + * @param object Plugin gst object. + * @param settings Settings struct to update. + * @param property_id Property id to update. + * @param value Property value. + * @param pspec Gst param type spec. + */ +void gst_projectm_base_get_property(GObject *object, + GstBaseProjectMSettings *settings, + guint property_id, GValue *value, + GParamSpec *pspec); + +/** + * Plugin init() delegate for projectM settings and priv. + * + * @param settings Settings to init. + * @param priv Private obj to init. + */ +void gst_projectm_base_init(GstBaseProjectMSettings *settings, + GstBaseProjectMPrivate *priv); + +/** + * Plugin finalize() delegate for projectM settings and priv. + * + * @param settings Settings to init. + * @param priv Private obj to init. + */ +void gst_projectm_base_finalize(GstBaseProjectMSettings *settings, + GstBaseProjectMPrivate *priv); + +/** + * GL start delegate to setup projectM fbo rendering. + * + * @param plugin Plugin gst object. + * @param priv Plugin priv data. + * @param settings Plugin settings. + * @param context The gl context to use for projectM rendering. + * @param vinfo Video rendering details. + * + * @return TRUE on success. + */ +gboolean gst_projectm_base_gl_start(GObject *plugin, + GstBaseProjectMPrivate *priv, + GstBaseProjectMSettings *settings, + GstGLContext *context, GstVideoInfo *vinfo); + +/** + * GL stop delegate to clean up projectM rendering resources. + * + * @param plugin Plugin gst object. + * @param priv Plugin priv data. + */ +void gst_projectm_base_gl_stop(GObject *plugin, GstBaseProjectMPrivate *priv); + +/** + * Just pushes audio data to projectM without rendering. + * + * @param priv Plugin priv data. + * @param in_audio Audio data buffer to push to projectM. + */ +void gst_projectm_base_fill_audio_buffer_unlocked(GstBaseProjectMPrivate *priv, + GstBuffer *in_audio); + +/** + * Render one frame with projectM. + * + * @param priv Plugin priv data. + * @param context ProjectM GL context. + * @param pts Current pts timestamp. + * @param in_audio Input audio buffer to push to projectM before rendering, may + * be NULL. + */ +void gst_projectm_base_fill_gl_memory_callback(GstBaseProjectMPrivate *priv, + GstGLContext *context, + GstGLFramebuffer *fbo, + GstClockTime pts, + GstBuffer *in_audio); + +/** + * Reset time offset for a new segment. + * + * @param priv Plugin priv data. + * @param pts_offset pts time offset for a new segment. + */ +void gst_projectm_base_set_segment_pts_offset(GstBaseProjectMPrivate *priv, + gint64 pts_offset); + +/** + * Install properties from projectM settings to given plugin class. + * + * @param gobject_class Plugin class to install properties to. + */ +void gst_projectm_base_install_properties(GObjectClass *gobject_class); + +/** + * Utility to parse a fraction from a string. + * + * @param str Fraction as string, ex. 60/1 + * @param numerator Return ref for numerator. + * @param denominator Return ref for denominator. + * + * @return TRUE if the fraction was parsed correctly. + */ +gboolean gst_projectm_base_parse_fraction(const gchar *str, gint *numerator, + gint *denominator); + +#define GST_PROJECTM_BASE_LOCK(priv) (g_mutex_lock(&priv->projectm_lock)) +#define GST_PROJECTM_BASE_UNLOCK(priv) (g_mutex_unlock(&priv->projectm_lock)) + +G_END_DECLS + +#endif // __GST_PROJECTM_BASE_H__ diff --git a/src/gstprojectmcaps.c b/src/gstprojectmcaps.c new file mode 100644 index 0000000..e146420 --- /dev/null +++ b/src/gstprojectmcaps.c @@ -0,0 +1,27 @@ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "gstprojectm.h" +#include "gstprojectmcaps.h" + +GST_DEBUG_CATEGORY_STATIC(gst_projectm_caps_debug); +#define GST_CAT_DEFAULT gst_projectm_caps_debug + +const gchar *get_audio_sink_cap() { + return GST_AUDIO_CAPS_MAKE("audio/x-raw, " + "format = (string) " GST_AUDIO_NE( + S16) ", " + "layout = (string) interleaved, " + "channels = (int) { 2 }, " + "rate = (int) { 44100 }, " + "channel-mask = (bitmask) { 0x0003 }"); +} + +const gchar *get_video_src_cap() { + return GST_VIDEO_CAPS_MAKE_WITH_FEATURES("memory:GLMemory", "RGBA"); +} diff --git a/src/caps.h b/src/gstprojectmcaps.h similarity index 55% rename from src/caps.h rename to src/gstprojectmcaps.h index 070d281..53237f7 100644 --- a/src/caps.h +++ b/src/gstprojectmcaps.h @@ -3,26 +3,24 @@ #include -#include "plugin.h" +#include "gstprojectm.h" G_BEGIN_DECLS /** * @brief Get audio sink caps based on the given type. * - * @param type - The type of audio caps to retrieve. * @return The audio caps format string. */ -const gchar *get_audio_sink_cap(unsigned int type); +const gchar *get_audio_sink_cap(); /** * Get video source caps based on the given type. * - * @param type - The type of video caps to retrieve. * @return The video caps format string. */ -const gchar *get_video_src_cap(unsigned int type); +const gchar *get_video_src_cap(); G_END_DECLS -#endif /* __GST_PROJECTM_CAPS_H__ */ \ No newline at end of file +#endif /* __GST_PROJECTM_CAPS_H__ */ diff --git a/src/gstprojectmconfig.h b/src/gstprojectmconfig.h new file mode 100644 index 0000000..7e3f5ff --- /dev/null +++ b/src/gstprojectmconfig.h @@ -0,0 +1,20 @@ +#ifndef __GST_PROJECTM_CONFIG_H__ +#define __GST_PROJECTM_CONFIG_H__ + +#include + +G_BEGIN_DECLS + +/** + * @brief Plugin Details + */ + +#define PACKAGE "GstProjectM" +#define PACKAGE_NAME "GstProjectM" +#define PACKAGE_VERSION "0.0.3" +#define PACKAGE_LICENSE "LGPL" +#define PACKAGE_ORIGIN "https://github.com/projectM-visualizer/gst-projectm" + +G_END_DECLS + +#endif /* __GST_PROJECTM_CONFIG_H__ */ diff --git a/src/plugin.c b/src/plugin.c deleted file mode 100644 index 125b2ed..0000000 --- a/src/plugin.c +++ /dev/null @@ -1,539 +0,0 @@ -#include -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#ifdef USE_GLEW -#include -#endif -#include -#include -#include - -#include - -#include "caps.h" -#include "config.h" -#include "debug.h" -#include "enums.h" -#include "gstglbaseaudiovisualizer.h" -#include "plugin.h" -#include "projectm.h" - -GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); -#define GST_CAT_DEFAULT gst_projectm_debug - -struct _GstProjectMPrivate { - GLenum gl_format; - projectm_handle handle; - - GstClockTime first_frame_time; - gboolean first_frame_received; -}; - -G_DEFINE_TYPE_WITH_CODE(GstProjectM, gst_projectm, - GST_TYPE_GL_BASE_AUDIO_VISUALIZER, - G_ADD_PRIVATE(GstProjectM) - GST_DEBUG_CATEGORY_INIT(gst_projectm_debug, - "gstprojectm", 0, - "Plugin Root")); - -void gst_projectm_set_property(GObject *object, guint property_id, - const GValue *value, GParamSpec *pspec) { - GstProjectM *plugin = GST_PROJECTM(object); - - const gchar *property_name = g_param_spec_get_name(pspec); - GST_DEBUG_OBJECT(plugin, "set-property <%s>", property_name); - - switch (property_id) { - case PROP_PRESET_PATH: - plugin->preset_path = g_strdup(g_value_get_string(value)); - break; - case PROP_TEXTURE_DIR_PATH: - plugin->texture_dir_path = g_strdup(g_value_get_string(value)); - break; - case PROP_BEAT_SENSITIVITY: - plugin->beat_sensitivity = g_value_get_float(value); - break; - case PROP_HARD_CUT_DURATION: - plugin->hard_cut_duration = g_value_get_double(value); - break; - case PROP_HARD_CUT_ENABLED: - plugin->hard_cut_enabled = g_value_get_boolean(value); - break; - case PROP_HARD_CUT_SENSITIVITY: - plugin->hard_cut_sensitivity = g_value_get_float(value); - break; - case PROP_SOFT_CUT_DURATION: - plugin->soft_cut_duration = g_value_get_double(value); - break; - case PROP_PRESET_DURATION: - plugin->preset_duration = g_value_get_double(value); - break; - case PROP_MESH_SIZE: { - const gchar *meshSizeStr = g_value_get_string(value); - gint width, height; - - gchar **parts = g_strsplit(meshSizeStr, ",", 2); - - if (parts && g_strv_length(parts) == 2) { - width = atoi(parts[0]); - height = atoi(parts[1]); - - plugin->mesh_width = width; - plugin->mesh_height = height; - - g_strfreev(parts); - } - } break; - case PROP_ASPECT_CORRECTION: - plugin->aspect_correction = g_value_get_boolean(value); - break; - case PROP_EASTER_EGG: - plugin->easter_egg = g_value_get_float(value); - break; - case PROP_PRESET_LOCKED: - plugin->preset_locked = g_value_get_boolean(value); - break; - case PROP_ENABLE_PLAYLIST: - plugin->enable_playlist = g_value_get_boolean(value); - break; - case PROP_SHUFFLE_PRESETS: - plugin->shuffle_presets = g_value_get_boolean(value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); - break; - } -} - -void gst_projectm_get_property(GObject *object, guint property_id, - GValue *value, GParamSpec *pspec) { - GstProjectM *plugin = GST_PROJECTM(object); - - const gchar *property_name = g_param_spec_get_name(pspec); - GST_DEBUG_OBJECT(plugin, "get-property <%s>", property_name); - - switch (property_id) { - case PROP_PRESET_PATH: - g_value_set_string(value, plugin->preset_path); - break; - case PROP_TEXTURE_DIR_PATH: - g_value_set_string(value, plugin->texture_dir_path); - break; - case PROP_BEAT_SENSITIVITY: - g_value_set_float(value, plugin->beat_sensitivity); - break; - case PROP_HARD_CUT_DURATION: - g_value_set_double(value, plugin->hard_cut_duration); - break; - case PROP_HARD_CUT_ENABLED: - g_value_set_boolean(value, plugin->hard_cut_enabled); - break; - case PROP_HARD_CUT_SENSITIVITY: - g_value_set_float(value, plugin->hard_cut_sensitivity); - break; - case PROP_SOFT_CUT_DURATION: - g_value_set_double(value, plugin->soft_cut_duration); - break; - case PROP_PRESET_DURATION: - g_value_set_double(value, plugin->preset_duration); - break; - case PROP_MESH_SIZE: { - gchar *meshSizeStr = - g_strdup_printf("%lu,%lu", plugin->mesh_width, plugin->mesh_height); - g_value_set_string(value, meshSizeStr); - g_free(meshSizeStr); - break; - } - case PROP_ASPECT_CORRECTION: - g_value_set_boolean(value, plugin->aspect_correction); - break; - case PROP_EASTER_EGG: - g_value_set_float(value, plugin->easter_egg); - break; - case PROP_PRESET_LOCKED: - g_value_set_boolean(value, plugin->preset_locked); - break; - case PROP_ENABLE_PLAYLIST: - g_value_set_boolean(value, plugin->enable_playlist); - break; - case PROP_SHUFFLE_PRESETS: - g_value_set_boolean(value, plugin->shuffle_presets); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); - break; - } -} - -static void gst_projectm_init(GstProjectM *plugin) { - plugin->priv = gst_projectm_get_instance_private(plugin); - - // Set default values for properties - plugin->preset_path = DEFAULT_PRESET_PATH; - plugin->texture_dir_path = DEFAULT_TEXTURE_DIR_PATH; - plugin->beat_sensitivity = DEFAULT_BEAT_SENSITIVITY; - plugin->hard_cut_duration = DEFAULT_HARD_CUT_DURATION; - plugin->hard_cut_enabled = DEFAULT_HARD_CUT_ENABLED; - plugin->hard_cut_sensitivity = DEFAULT_HARD_CUT_SENSITIVITY; - plugin->soft_cut_duration = DEFAULT_SOFT_CUT_DURATION; - plugin->preset_duration = DEFAULT_PRESET_DURATION; - plugin->enable_playlist = DEFAULT_ENABLE_PLAYLIST; - plugin->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; - - const gchar *meshSizeStr = DEFAULT_MESH_SIZE; - gint width, height; - - gchar **parts = g_strsplit(meshSizeStr, ",", 2); - - if (parts && g_strv_length(parts) == 2) { - width = atoi(parts[0]); - height = atoi(parts[1]); - - plugin->mesh_width = width; - plugin->mesh_height = height; - - g_strfreev(parts); - } - - plugin->aspect_correction = DEFAULT_ASPECT_CORRECTION; - plugin->easter_egg = DEFAULT_EASTER_EGG; - plugin->preset_locked = DEFAULT_PRESET_LOCKED; - plugin->priv->handle = NULL; -} - -static void gst_projectm_finalize(GObject *object) { - GstProjectM *plugin = GST_PROJECTM(object); - g_free(plugin->preset_path); - g_free(plugin->texture_dir_path); - G_OBJECT_CLASS(gst_projectm_parent_class)->finalize(object); -} - -static void gst_projectm_gl_stop(GstGLBaseAudioVisualizer *src) { - GstProjectM *plugin = GST_PROJECTM(src); - if (plugin->priv->handle) { - GST_DEBUG_OBJECT(plugin, "Destroying ProjectM instance"); - projectm_destroy(plugin->priv->handle); - plugin->priv->handle = NULL; - } -} - -static gboolean gst_projectm_gl_start(GstGLBaseAudioVisualizer *glav) { - // Cast the audio visualizer to the ProjectM plugin - GstProjectM *plugin = GST_PROJECTM(glav); - -#ifdef USE_GLEW - GST_DEBUG_OBJECT(plugin, "Initializing GLEW"); - GLenum err = glewInit(); - if (GLEW_OK != err) { - GST_ERROR_OBJECT(plugin, "GLEW initialization failed"); - return FALSE; - } -#endif - - // Check if ProjectM instance exists, and create if not - if (!plugin->priv->handle) { - // Create ProjectM instance - plugin->priv->handle = projectm_init(plugin); - if (!plugin->priv->handle) { - GST_ERROR_OBJECT(plugin, "ProjectM could not be initialized"); - return FALSE; - } - gl_error_handler(glav->context, plugin); - } - - return TRUE; -} - -static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav) { - GstAudioVisualizer *bscope = GST_AUDIO_VISUALIZER(glav); - GstProjectM *plugin = GST_PROJECTM(glav); - - // Calculate depth based on pixel stride and bits - gint depth = bscope->vinfo.finfo->pixel_stride[0] * - ((bscope->vinfo.finfo->bits >= 8) ? 8 : 1); - - // Calculate required samples per frame - bscope->req_spf = - (bscope->ainfo.channels * bscope->ainfo.rate * 2) / bscope->vinfo.fps_n; - - // get GStreamer video format and map it to the corresponding OpenGL pixel - // format - const GstVideoFormat video_format = GST_VIDEO_INFO_FORMAT(&bscope->vinfo); - - // TODO: why is the reversed byte order needed when copying pixel data from - // OpenGL ? - switch (video_format) { - case GST_VIDEO_FORMAT_ABGR: - plugin->priv->gl_format = GL_RGBA; - break; - - case GST_VIDEO_FORMAT_RGBA: - // GL_ABGR_EXT does not seem to be well-supported, does not work on Windows - plugin->priv->gl_format = GL_ABGR_EXT; - break; - - default: - GST_ERROR_OBJECT(plugin, "Unsupported video format: %d", video_format); - return FALSE; - } - - // Log audio info - GST_DEBUG_OBJECT( - glav, "Audio Information ", - bscope->ainfo.channels, bscope->ainfo.rate, - bscope->ainfo.finfo->description); - - // Log video info - GST_DEBUG_OBJECT(glav, - "Video Information ", - GST_VIDEO_INFO_WIDTH(&bscope->vinfo), - GST_VIDEO_INFO_HEIGHT(&bscope->vinfo), bscope->vinfo.fps_n, - bscope->vinfo.fps_d, depth, bscope->req_spf); - - return TRUE; -} - -static double get_seconds_since_first_frame(GstProjectM *plugin, - GstVideoFrame *frame) { - if (!plugin->priv->first_frame_received) { - // Store the timestamp of the first frame - plugin->priv->first_frame_time = GST_BUFFER_PTS(frame->buffer); - plugin->priv->first_frame_received = TRUE; - return 0.0; - } - - // Calculate elapsed time - GstClockTime current_time = GST_BUFFER_PTS(frame->buffer); - GstClockTime elapsed_time = current_time - plugin->priv->first_frame_time; - - // Convert to fractional seconds - gdouble elapsed_seconds = (gdouble)elapsed_time / GST_SECOND; - - return elapsed_seconds; -} - -// TODO: CLEANUP & ADD DEBUGGING -static gboolean gst_projectm_render(GstGLBaseAudioVisualizer *glav, - GstBuffer *audio, GstVideoFrame *video) { - GstProjectM *plugin = GST_PROJECTM(glav); - - GstMapInfo audioMap; - gboolean result = TRUE; - - // get current gst (PTS) time and set projectM time - double seconds_since_first_frame = - get_seconds_since_first_frame(plugin, video); - projectm_set_frame_time(plugin->priv->handle, seconds_since_first_frame); - - // AUDIO - gst_buffer_map(audio, &audioMap, GST_MAP_READ); - - // GST_DEBUG_OBJECT(plugin, "Audio Samples: %u, Offset: %lu, Offset End: %lu, - // Sample Rate: %d, FPS: %d, Required Samples Per Frame: %d", - // audioMap.size / 8, audio->offset, audio->offset_end, - // bscope->ainfo.rate, bscope->vinfo.fps_n, bscope->req_spf); - - projectm_pcm_add_int16(plugin->priv->handle, (gint16 *)audioMap.data, - audioMap.size / 4, PROJECTM_STEREO); - - // GST_DEBUG_OBJECT(plugin, "Audio Data: %d %d %d %d", ((gint16 - // *)audioMap.data)[100], ((gint16 *)audioMap.data)[101], ((gint16 - // *)audioMap.data)[102], ((gint16 *)audioMap.data)[103]); - - // VIDEO - const GstGLFuncs *glFunctions = glav->context->gl_vtable; - - size_t windowWidth, windowHeight; - - projectm_get_window_size(plugin->priv->handle, &windowWidth, &windowHeight); - - projectm_opengl_render_frame(plugin->priv->handle); - gl_error_handler(glav->context, plugin); - - glFunctions->ReadPixels(0, 0, windowWidth, windowHeight, - plugin->priv->gl_format, GL_UNSIGNED_INT_8_8_8_8, - (guint8 *)GST_VIDEO_FRAME_PLANE_DATA(video, 0)); - - gst_buffer_unmap(audio, &audioMap); - - // GST_DEBUG_OBJECT(plugin, "Video Data: %d %d\n", - // GST_VIDEO_FRAME_N_PLANES(video), ((uint8_t - // *)(GST_VIDEO_FRAME_PLANE_DATA(video, 0)))[0]); - - // GST_DEBUG_OBJECT(plugin, "Rendered one frame"); - - return result; -} - -static void gst_projectm_class_init(GstProjectMClass *klass) { - GObjectClass *gobject_class = (GObjectClass *)klass; - GstElementClass *element_class = (GstElementClass *)klass; - GstGLBaseAudioVisualizerClass *scope_class = - GST_GL_BASE_AUDIO_VISUALIZER_CLASS(klass); - - // Setup audio and video caps - const gchar *audio_sink_caps = get_audio_sink_cap(0); - const gchar *video_src_caps = get_video_src_cap(0); - - gst_element_class_add_pad_template( - GST_ELEMENT_CLASS(klass), - gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, - gst_caps_from_string(video_src_caps))); - gst_element_class_add_pad_template( - GST_ELEMENT_CLASS(klass), - gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - gst_caps_from_string(audio_sink_caps))); - - gst_element_class_set_static_metadata( - GST_ELEMENT_CLASS(klass), "ProjectM Visualizer", "Generic", - "A plugin for visualizing music using ProjectM", - "AnomieVision | Tristan Charpentier " - ""); - - // Setup properties - gobject_class->set_property = gst_projectm_set_property; - gobject_class->get_property = gst_projectm_get_property; - - g_object_class_install_property( - gobject_class, PROP_PRESET_PATH, - g_param_spec_string( - "preset", "Preset", - "Specifies the path to the preset file. The preset file determines " - "the visual style and behavior of the audio visualizer.", - DEFAULT_PRESET_PATH, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_TEXTURE_DIR_PATH, - g_param_spec_string("texture-dir", "Texture Directory", - "Sets the path to the directory containing textures " - "used in the visualizer.", - DEFAULT_TEXTURE_DIR_PATH, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_BEAT_SENSITIVITY, - g_param_spec_float( - "beat-sensitivity", "Beat Sensitivity", - "Controls the sensitivity to audio beats. Higher values make the " - "visualizer respond more strongly to beats.", - 0.0, 5.0, DEFAULT_BEAT_SENSITIVITY, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_HARD_CUT_DURATION, - g_param_spec_double("hard-cut-duration", "Hard Cut Duration", - "Sets the duration, in seconds, for hard cuts. Hard " - "cuts are abrupt transitions in the visualizer.", - 0.0, 999999.0, DEFAULT_HARD_CUT_DURATION, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_HARD_CUT_ENABLED, - g_param_spec_boolean( - "hard-cut-enabled", "Hard Cut Enabled", - "Enables or disables hard cuts. When enabled, the visualizer may " - "exhibit sudden transitions based on the audio input.", - DEFAULT_HARD_CUT_ENABLED, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_HARD_CUT_SENSITIVITY, - g_param_spec_float( - "hard-cut-sensitivity", "Hard Cut Sensitivity", - "Adjusts the sensitivity of the visualizer to hard cuts. Higher " - "values increase the responsiveness to abrupt changes in audio.", - 0.0, 1.0, DEFAULT_HARD_CUT_SENSITIVITY, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_SOFT_CUT_DURATION, - g_param_spec_double( - "soft-cut-duration", "Soft Cut Duration", - "Sets the duration, in seconds, for soft cuts. Soft cuts are " - "smoother transitions between visualizer states.", - 0.0, 999999.0, DEFAULT_SOFT_CUT_DURATION, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_PRESET_DURATION, - g_param_spec_double("preset-duration", "Preset Duration", - "Sets the duration, in seconds, for each preset. A " - "zero value causes the preset to play indefinitely.", - 0.0, 999999.0, DEFAULT_PRESET_DURATION, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_MESH_SIZE, - g_param_spec_string("mesh-size", "Mesh Size", - "Sets the size of the mesh used in rendering. The " - "format is 'width,height'.", - DEFAULT_MESH_SIZE, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_ASPECT_CORRECTION, - g_param_spec_boolean( - "aspect-correction", "Aspect Correction", - "Enables or disables aspect ratio correction. When enabled, the " - "visualizer adjusts for aspect ratio differences in rendering.", - DEFAULT_ASPECT_CORRECTION, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_EASTER_EGG, - g_param_spec_float( - "easter-egg", "Easter Egg", - "Controls the activation of an Easter Egg feature. The value " - "determines the likelihood of triggering the Easter Egg.", - 0.0, 1.0, DEFAULT_EASTER_EGG, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_PRESET_LOCKED, - g_param_spec_boolean( - "preset-locked", "Preset Locked", - "Locks or unlocks the current preset. When locked, the visualizer " - "remains on the current preset without automatic changes.", - DEFAULT_PRESET_LOCKED, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_ENABLE_PLAYLIST, - g_param_spec_boolean( - "enable-playlist", "Enable Playlist", - "Enables or disables the playlist feature. When enabled, the " - "visualizer can switch between presets based on a provided playlist.", - DEFAULT_ENABLE_PLAYLIST, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property( - gobject_class, PROP_SHUFFLE_PRESETS, - g_param_spec_boolean( - "shuffle-presets", "Shuffle Presets", - "Enables or disables preset shuffling. When enabled, the visualizer " - "randomly selects presets from the playlist if presets are provided " - "and not locked. Playlist must be enabled for this to take effect.", - DEFAULT_SHUFFLE_PRESETS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - gobject_class->finalize = gst_projectm_finalize; - - scope_class->supported_gl_api = GST_GL_API_OPENGL3 | GST_GL_API_GLES2; - scope_class->gl_start = GST_DEBUG_FUNCPTR(gst_projectm_gl_start); - scope_class->gl_stop = GST_DEBUG_FUNCPTR(gst_projectm_gl_stop); - scope_class->gl_render = GST_DEBUG_FUNCPTR(gst_projectm_render); - scope_class->setup = GST_DEBUG_FUNCPTR(gst_projectm_setup); -} - -static gboolean plugin_init(GstPlugin *plugin) { - GST_DEBUG_CATEGORY_INIT(gst_projectm_debug, "projectm", 0, - "projectM visualizer plugin"); - - return gst_element_register(plugin, "projectm", GST_RANK_NONE, - GST_TYPE_PROJECTM); -} - -GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, projectm, - "plugin to visualize audio using the ProjectM library", - plugin_init, PACKAGE_VERSION, PACKAGE_LICENSE, PACKAGE_NAME, - PACKAGE_ORIGIN) diff --git a/src/projectm.c b/src/projectm.c deleted file mode 100644 index 1bac137..0000000 --- a/src/projectm.c +++ /dev/null @@ -1,127 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include - -#include -#include - -#include "plugin.h" -#include "projectm.h" - -GST_DEBUG_CATEGORY_STATIC(projectm_debug); -#define GST_CAT_DEFAULT projectm_debug - -projectm_handle projectm_init(GstProjectM *plugin) { - projectm_handle handle = NULL; - projectm_playlist_handle playlist = NULL; - - GST_DEBUG_CATEGORY_INIT(projectm_debug, "projectm", 0, "ProjectM"); - - GstAudioVisualizer *bscope = GST_AUDIO_VISUALIZER(plugin); - - // Create ProjectM instance - GST_DEBUG_OBJECT(plugin, "Creating projectM instance.."); - handle = projectm_create(); - - if (!handle) { - GST_DEBUG_OBJECT( - plugin, - "project_create() returned NULL, projectM instance was not created!"); - return NULL; - } else { - GST_DEBUG_OBJECT(plugin, "Created projectM instance!"); - } - - if (plugin->enable_playlist) { - GST_DEBUG_OBJECT(plugin, "Playlist enabled"); - - // initialize preset playlist - playlist = projectm_playlist_create(handle); - projectm_playlist_set_shuffle(playlist, plugin->shuffle_presets); - // projectm_playlist_set_preset_switched_event_callback(_playlist, - // &ProjectMWrapper::PresetSwitchedEvent, static_cast(this)); - } else { - GST_DEBUG_OBJECT(plugin, "Playlist disabled"); - } - - // Log properties - GST_INFO_OBJECT( - plugin, - "Using Properties: " - "preset=%s, " - "texture-dir=%s, " - "beat-sensitivity=%f, " - "hard-cut-duration=%f, " - "hard-cut-enabled=%d, " - "hard-cut-sensitivity=%f, " - "soft-cut-duration=%f, " - "preset-duration=%f, " - "mesh-size=(%lu, %lu)" - "aspect-correction=%d, " - "easter-egg=%f, " - "preset-locked=%d, " - "enable-playlist=%d, " - "shuffle-presets=%d", - plugin->preset_path, plugin->texture_dir_path, plugin->beat_sensitivity, - plugin->hard_cut_duration, plugin->hard_cut_enabled, - plugin->hard_cut_sensitivity, plugin->soft_cut_duration, - plugin->preset_duration, plugin->mesh_width, plugin->mesh_height, - plugin->aspect_correction, plugin->easter_egg, plugin->preset_locked, - plugin->enable_playlist, plugin->shuffle_presets); - - // Load preset file if path is provided - if (plugin->preset_path != NULL) { - int added_count = - projectm_playlist_add_path(playlist, plugin->preset_path, true, false); - GST_INFO("Loaded preset path: %s, presets found: %d", plugin->preset_path, - added_count); - } - - // Set texture search path if directory path is provided - if (plugin->texture_dir_path != NULL) { - const gchar *texturePaths[1] = {plugin->texture_dir_path}; - projectm_set_texture_search_paths(handle, texturePaths, 1); - } - - // Set properties - projectm_set_beat_sensitivity(handle, plugin->beat_sensitivity); - projectm_set_hard_cut_duration(handle, plugin->hard_cut_duration); - projectm_set_hard_cut_enabled(handle, plugin->hard_cut_enabled); - projectm_set_hard_cut_sensitivity(handle, plugin->hard_cut_sensitivity); - projectm_set_soft_cut_duration(handle, plugin->soft_cut_duration); - - // Set preset duration, or set to in infinite duration if zero - if (plugin->preset_duration > 0.0) { - projectm_set_preset_duration(handle, plugin->preset_duration); - - // kick off the first preset - if (projectm_playlist_size(playlist) > 1 && !plugin->preset_locked) { - projectm_playlist_play_next(playlist, true); - } - } else { - projectm_set_preset_duration(handle, 999999.0); - } - - projectm_set_mesh_size(handle, plugin->mesh_width, plugin->mesh_height); - projectm_set_aspect_correction(handle, plugin->aspect_correction); - projectm_set_easter_egg(handle, plugin->easter_egg); - projectm_set_preset_locked(handle, plugin->preset_locked); - - projectm_set_fps(handle, GST_VIDEO_INFO_FPS_N(&bscope->vinfo)); - projectm_set_window_size(handle, GST_VIDEO_INFO_WIDTH(&bscope->vinfo), - GST_VIDEO_INFO_HEIGHT(&bscope->vinfo)); - - return handle; -} - -// void projectm_render(GstProjectM *plugin, gint16 *samples, gint sample_count) -// { -// GST_DEBUG_OBJECT(plugin, "Rendering %d samples", sample_count); - -// projectm_pcm_add_int16(plugin->handle, samples, sample_count, -// PROJECTM_STEREO); - -// projectm_opengl_render_frame(plugin->handle); -// } diff --git a/src/projectm.h b/src/projectm.h deleted file mode 100644 index 1ba6a37..0000000 --- a/src/projectm.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __PROJECTM_H__ -#define __PROJECTM_H__ - -#include - -#include "plugin.h" -#include - -G_BEGIN_DECLS - -/** - * @brief Initialize ProjectM - */ -projectm_handle projectm_init(GstProjectM *plugin); - -/** - * @brief Render ProjectM - */ -// void projectm_render(GstProjectM *plugin, gint16 *samples, gint -// sample_count); - -G_END_DECLS - -#endif /* __PROJECTM_H__ */ \ No newline at end of file diff --git a/src/register.c b/src/register.c new file mode 100644 index 0000000..09f9238 --- /dev/null +++ b/src/register.c @@ -0,0 +1,38 @@ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstprojectm.h" +#include "gstprojectmconfig.h" + +#include + +/* + * This unit registers all gst elements from this plugin library to make them + * available to GStreamer. + */ + +GST_DEBUG_CATEGORY(gst_projectm_debug); +#define GST_CAT_DEFAULT gst_projectm_debug + +static gboolean plugin_init(GstPlugin *plugin) { + + gst_projectm_base_init_once(); + + GST_DEBUG_CATEGORY_INIT(gst_projectm_debug, "projectm", 0, + "projectM visualizer plugin"); + + // register main plugin projectM element + gboolean p1 = gst_element_register(plugin, "projectm", GST_RANK_NONE, + GST_TYPE_PROJECTM); + + // add additional elements here.. + + return p1; +} + +GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, projectm, + "plugin to visualize audio using the ProjectM library", + plugin_init, PACKAGE_VERSION, PACKAGE_LICENSE, PACKAGE_NAME, + PACKAGE_ORIGIN) diff --git a/src/renderbuffer.c b/src/renderbuffer.c new file mode 100644 index 0000000..96c006d --- /dev/null +++ b/src/renderbuffer.c @@ -0,0 +1,653 @@ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "renderbuffer.h" + +GST_DEBUG_CATEGORY_STATIC(renderbuffer_debug); +#define GST_CAT_DEFAULT renderbuffer_debug + +/** + * Queue shutdown signal token. + */ +static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; + +/** + * Number of frames inspected by EMA. + */ +#ifndef RB_EMA_FPS_ADJUST_INTERVAL +#define RB_EMA_FPS_ADJUST_INTERVAL 10 +#endif + +/** + * EMA alpha = 0.25 + */ +#ifndef RB_EMA_ALPHA_N +#define RB_EMA_ALPHA_N 1 +#define RB_EMA_ALPHA_D 4 +#endif + +/** + * Increase frame duration (slow down fps) in case of detected lag. + * +20% + */ +#ifndef RB_EMA_FRAME_DURATION_INCREASE_N +#define RB_EMA_FRAME_DURATION_INCREASE_N 12 +#define RB_EMA_FRAME_DURATION_INCREASE_D 10 +#endif + +/** + * Decrease frame duration (speed up fps) in case rendering performance + * recovers. -5% + */ +#ifndef RB_EMA_FRAME_DURATION_DECREASE_N +#define RB_EMA_FRAME_DURATION_DECREASE_N 95 +#define RB_EMA_FRAME_DURATION_DECREASE_D 100 +#endif + +/** + * Tolerance for being too slow. + * Allow render time up to 1.1x + */ +#ifndef RB_EMA_FRAME_DURATION_TOLERANCE_UP_N +#define RB_EMA_FRAME_DURATION_TOLERANCE_UP_N 110 +#define RB_EMA_FRAME_DURATION_TOLERANCE_UP_D 100 +#endif + +/** + * Tolerance for being too fast. + * allow render time as low as 0.9x + */ +#ifndef RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N +#define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N 9 +#define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D 10 +#endif + +/** + * Possible option to drop frames that are too late after rendering if they + * would be dropped by a downstream sink anyway. + * Experimental, tends to increase flicker in most cases. Disabled per default. + */ +// #define RB_DROP_LATE_FRAMES +#ifndef RB_DROP_LATE_FRAMES_TOLERANCE +#define RB_DROP_LATE_FRAMES_TOLERANCE (GST_MSECOND * 8) +#endif + +/** + * How much time has to be left of the time budget for scheduling before + * entering wait. Tolerance to account for scheduling overhead etc. to guarantee + * a defined max run-time of the scheduling process. + */ +#ifndef MIN_FREE_SLOT_SCHEDULE_WAIT +#define MIN_FREE_SLOT_SCHEDULE_WAIT (GST_MSECOND * 1) +#endif + +/** + * Exponential Moving Average (EMA)-based adaptive frame duration (fps) + * adjustment. Determines desired frame duration change based on the + * frame render duration and min/max fps configs. + * + * @param state Render state data. + * @param render_duration Render duration for the last frame in nanos. + * @param frame_duration Current desired frame duration in nanos (fps * + * GST_SECOND). + */ +static void rb_handle_adaptive_fps_ema(RBRenderBuffer *state, + const GstClockTime render_duration, + const GstClockTime frame_duration) { + state->frame_counter++; + + // EMA smoothing: smoothed = alpha * x + (1 - alpha) * prev + state->smoothed_render_time = + + gst_util_uint64_scale_int(render_duration, RB_EMA_ALPHA_N, + RB_EMA_ALPHA_D) + + + gst_util_uint64_scale_int(state->smoothed_render_time, + RB_EMA_ALPHA_D - RB_EMA_ALPHA_N, + RB_EMA_ALPHA_D); + + if (state->frame_counter >= RB_EMA_FPS_ADJUST_INTERVAL) { + + GstClockTime new_duration; + state->frame_counter = 0; + + const GstClockTime upper_threshold = gst_util_uint64_scale_int( + frame_duration, RB_EMA_FRAME_DURATION_TOLERANCE_UP_N, + RB_EMA_FRAME_DURATION_TOLERANCE_UP_D); + + const GstClockTime lower_threshold = gst_util_uint64_scale_int( + frame_duration, RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N, + RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D); + + if (state->smoothed_render_time > upper_threshold) { + + // rendering too slow, increase frame duration (drop FPS) + new_duration = gst_util_uint64_scale_int( + frame_duration, RB_EMA_FRAME_DURATION_INCREASE_N, + RB_EMA_FRAME_DURATION_INCREASE_D); + } else if (state->smoothed_render_time < lower_threshold) { + + // rendering fast enough, try to decrease frame duration (increase FPS) + new_duration = gst_util_uint64_scale_int( + frame_duration, RB_EMA_FRAME_DURATION_DECREASE_N, + RB_EMA_FRAME_DURATION_DECREASE_D); + } else { + // within tolerance, no change + return; + } + + g_mutex_lock(&state->slot_lock); + + // clamp min/max frame duration (fps) according to config + if (new_duration > state->max_frame_duration) { + new_duration = state->max_frame_duration; + } else if (new_duration < state->caps_frame_duration) { + new_duration = state->caps_frame_duration; + } + + g_mutex_unlock(&state->slot_lock); + + if (new_duration != frame_duration) { + GST_DEBUG_OBJECT( + state->plugin, + "Adaptive FPS: frame duration changed from %" GST_TIME_FORMAT + " to %" GST_TIME_FORMAT, + GST_TIME_ARGS(frame_duration), GST_TIME_ARGS(new_duration)); + + // pass new frame duration to callback + state->adjust_fps_func(state, new_duration); + } + } +} + +static void rb_queue_gl_buffer_cleanup(const RBRenderBuffer *state, + GstBuffer *out) { + g_async_queue_push(state->buffer_cleanup_queue, out); +} + +void rb_set_qos_enabled(RBRenderBuffer *state, const gboolean is_qos_enabled) { + g_mutex_lock(&state->slot_lock); + state->qos_enabled = is_qos_enabled; + g_mutex_unlock(&state->slot_lock); +} + +void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, + const GstGLContextThreadFunc gl_fill_func, + const RBAdjustFpsFunc adjust_fps_func, + const GstClockTime max_frame_duration, + const GstClockTime caps_frame_duration, + const gboolean is_qos_enabled) { + + GST_DEBUG_CATEGORY_INIT(renderbuffer_debug, "renderbuffer", 0, + "projectM visualizer plugin render buffer"); + + state->plugin = plugin; + state->adjust_fps_func = adjust_fps_func; + + state->gl_context = NULL; + state->src_pad = NULL; + state->render_thread = NULL; + state->running = FALSE; + + state->qos_enabled = is_qos_enabled; + state->caps_frame_duration = caps_frame_duration; + state->max_frame_duration = max_frame_duration; + + state->last_insert_index = -1; + state->last_render_index = -1; + state->frame_counter = 0; + state->smoothed_render_time = 0; + + g_mutex_init(&state->slot_lock); + g_cond_init(&state->slot_available_cond); + g_cond_init(&state->render_queued_cond); + g_cond_init(&state->render_complete_cond); + state->buffer_cleanup_queue = g_async_queue_new(); + + for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { + state->slots[i].state = RB_EMPTY; + state->slots[i].plugin = plugin; + state->slots[i].gl_result = FALSE; + state->slots[i].pts = GST_CLOCK_TIME_NONE; + state->slots[i].frame_duration = 0; + state->slots[i].latency = GST_CLOCK_TIME_NONE; + state->slots[i].running_time = GST_CLOCK_TIME_NONE; + state->slots[i].out_buf = NULL; + state->slots[i].gl_fill_func = gl_fill_func; + state->slots[i].in_audio = NULL; + } +} + +void rb_dispose_render_buffer(RBRenderBuffer *state) { + g_async_queue_unref(state->buffer_cleanup_queue); + g_cond_clear(&state->slot_available_cond); + g_cond_clear(&state->render_queued_cond); + g_cond_clear(&state->render_complete_cond); + g_mutex_clear(&state->slot_lock); +} + +RBQueueResult rb_queue_render_job(RBQueueArgs *args) { + + RBRenderBuffer *state = args->render_buffer; + const gboolean wait_is_limited = args->max_wait != GST_CLOCK_TIME_NONE; + const GstClockTime start = gst_util_get_timestamp(); + GstClockTimeDiff used_wait = 0; + + g_mutex_lock(&state->slot_lock); + + RBSlot *slot = NULL; + gint slot_index; + + gboolean found_slot = FALSE; + while (!found_slot) { + + // next slot to insert to + slot_index = (state->last_insert_index + 1) % NUM_RENDER_SLOTS; + slot = &state->slots[slot_index]; + + // jump over busy slot that's currently rendering if needed + if (slot->state == RB_BUSY) { + slot_index = (state->last_insert_index + 2) % NUM_RENDER_SLOTS; + } + + // in case there is only one slot, it may still be busy + found_slot = + slot->state != RB_BUSY && (wait_is_limited || slot->state == RB_EMPTY); + + if (!found_slot) { + if (wait_is_limited) { + const GstClockTimeDiff remaining_wait = args->max_wait - used_wait; + // not waiting until the very last millisecond + // to avoid exceeding time budget + if (remaining_wait > MIN_FREE_SLOT_SCHEDULE_WAIT) { + + // this is in microseconds for a change + const gint64 now = g_get_monotonic_time(); + const gint64 deadline = now + remaining_wait / 1000; + + g_cond_wait_until(&state->slot_available_cond, &state->slot_lock, + deadline); + + used_wait = (GstClockTimeDiff)gst_util_get_timestamp() - start; + } else { + // not enough time left + break; + } + } else { + // no time constraints, frames are never dropped + // we just wait and keep trying + g_cond_wait(&state->slot_available_cond, &state->slot_lock); + } + } + } + + if (slot->state == RB_BUSY) { + // out of time, and we still can't schedule + g_mutex_unlock(&state->slot_lock); + return RB_TIMEOUT; + } + + state->last_insert_index = slot_index; + + // evict if already in use and clear buffers + const gboolean is_evicted = slot->state == RB_READY; + + if (slot->in_audio != NULL) { + gst_buffer_unref(slot->in_audio); + } + + if (slot->out_buf != NULL) { + rb_queue_gl_buffer_cleanup(state, slot->out_buf); + slot->out_buf = NULL; + } + + // populate slot + slot->state = RB_READY; + slot->gl_result = FALSE; + slot->pts = args->pts; + slot->frame_duration = args->frame_duration; + slot->latency = args->latency; + slot->running_time = args->running_time; + slot->in_audio = gst_buffer_copy_deep(args->in_audio); + + // signal render thread that there is something to do + g_cond_signal(&state->render_queued_cond); + + if (args->sync_rendering) { + // block until rendering completed, if requested + g_cond_wait(&state->render_complete_cond, &state->slot_lock); + } + + g_mutex_unlock(&state->slot_lock); + + RBQueueResult result; + if (found_slot) { + result = is_evicted == FALSE ? RB_SUCCESS : RB_EVICTED; + } else { + result = RB_TIMEOUT; + } + return result; +} + +void rb_queue_render_job_warn(RBQueueArgs *args) { + + RBRenderBuffer *state = args->render_buffer; + const GstClockTime start_ts = gst_util_get_timestamp(); + + const RBQueueResult result = rb_queue_render_job(args); + + switch (result) { + case RB_EVICTED: { + GST_DEBUG_OBJECT(state->plugin, + "Dropping previous GL frame from render buffer, " + "it was not picked up for rendering in time (evicted). " + "max-wait: %" GST_TIME_FORMAT ", pts: %" GST_TIME_FORMAT, + GST_TIME_ARGS(args->max_wait), GST_TIME_ARGS(args->pts)); + break; + } + + case RB_TIMEOUT: { + const GstClockTime now = gst_util_get_timestamp(); + GST_DEBUG_OBJECT( + state->plugin, + "Dropping GL frame from render buffer, waiting for free slot took too " + "long. elapsed: %" GST_TIME_FORMAT ", max-wait: %" GST_TIME_FORMAT + ", pts: %" GST_TIME_FORMAT, + GST_TIME_ARGS(now - start_ts), GST_TIME_ARGS(args->max_wait), + GST_TIME_ARGS(args->pts)); + break; + } + + case RB_SUCCESS: + break; + } +} + +/** + * Calculate current time based on given element's clock for QoS checks. + * + * @param element The plugin element. + * @return Current time as determined by clock used by element. + */ +static GstClockTime rb_element_render_time(GstElement *element) { + const GstClockTime base_time = gst_element_get_base_time(element); + GstClock *clock = gst_element_get_clock(element); + const GstClockTime now = gst_clock_get_time(clock); + return now - base_time; +} + +/** + * Determine if it's likely too late push a buffer, as it would likely be + * dropped by a pipeline synchronized sink. + * + * @param element The plugin element. + * @param latency Pipeline latency. + * @param running_time Current buffer running time. + * @return TRUE in case the buffer is too late. + */ +static gboolean rb_is_render_too_late(GstElement *element, + const GstClockTime latency, + const GstClockTime running_time) { + + if (latency == GST_CLOCK_TIME_NONE) { + return FALSE; + } + + const GstClockTime tolerance = RB_DROP_LATE_FRAMES_TOLERANCE; + + const GstClockTime render_time = rb_element_render_time(element); + + // latest time to push this buffer for it to make it to sink in time + const GstClockTime latest_push_time = running_time + latency; + + if (render_time > latest_push_time + tolerance) { + GST_DEBUG_OBJECT(element, + "Dropping late frame: render_time %" GST_TIME_FORMAT + " > buffer_running_time %" GST_TIME_FORMAT + " + latency %" GST_TIME_FORMAT + " + slack %" GST_TIME_FORMAT, + GST_TIME_ARGS(render_time), GST_TIME_ARGS(running_time), + GST_TIME_ARGS(latency), GST_TIME_ARGS(tolerance)); + return TRUE; + } + return FALSE; +} + +/** + * Callback for scheduling gl buffer release with gl thread. + * + * @param context Current gl context. + * @param buf GL buffer to release. + */ +static void gl_buffer_cleanup(GstGLContext *context, gpointer buf) { + gst_buffer_unref(GST_BUFFER(buf)); +} + +/** + * Used to dispose of gl buffers only. + * Consume buffers to clean-up and dispatches release through gl thread. + * + * @param user_data The GstBuffer pointer to release. + * @return + */ +static gpointer cleanup_thread_func(gpointer user_data) { + + const RBRenderBuffer *state = (RBRenderBuffer *)user_data; + while (state->running) { + + gpointer item = g_async_queue_pop(state->buffer_cleanup_queue); + + if (!item || item == RB_Q_SHUTDOWN_SIGNAL) + continue; + + gst_gl_context_thread_add(state->gl_context, gl_buffer_cleanup, item); + } + return NULL; +} + +/** + * Render thread main worker function. + * + * @param user_data Render buffer to work on. + * @return NULL + */ +static gpointer rb_render_thread_func(gpointer user_data) { + + RBRenderBuffer *state = (RBRenderBuffer *)user_data; +#if NUM_RENDER_SLOTS > 2 + GstClockTime last_pts = 0; +#endif + // slot modifications are locked + g_mutex_lock(&state->slot_lock); + + // start working on rendering frames until we shut down + while (state->running) { + + // first find a slot with data that's ready to render + gboolean found_slot = FALSE; + RBSlot *slot; + gint render_index; + while (!found_slot) { + render_index = (state->last_render_index + 1) % NUM_RENDER_SLOTS; + + slot = &state->slots[render_index]; + + // find a slot with audio input data + // also check if it's already older than the last frame or if it's the + // first frame (shouldn't happen unless the ring buffer capacity > 2) + if (slot->state == RB_READY +#if NUM_RENDER_SLOTS > 2 + // wontfix: segment events would need to be handled for this check to + // work right otherwise last_pts is not reset when the pts changes. + && (last_pts == 0 || slot->pts > last_pts) +#endif + ) { + found_slot = TRUE; + } else { + // no data is ready, wait for a new audio buffer being pushed + g_cond_wait(&state->render_queued_cond, &state->slot_lock); + if (state->running == FALSE) { + break; + } + } + } + + // no slot means we're not running anymore + if (found_slot == FALSE) { + g_cond_signal(&state->render_complete_cond); + break; + } + + // update iteration maker + state->last_render_index = render_index; +#if NUM_RENDER_SLOTS > 2 + last_pts = slot->pts; +#endif + + // nobody else is allowed to touch the slot anymore, it's owned by the + // render thread now + slot->state = RB_BUSY; + + g_mutex_unlock(&state->slot_lock); + + // measure rendering for QoS + GstClockTime render_start = gst_util_get_timestamp(); + + // Dispatch slot to GL thread + gst_gl_context_thread_add(state->gl_context, slot->gl_fill_func, slot); + + GstClockTime render_time = gst_util_get_timestamp() - render_start; + + // render took longer than the frame duration, this is a problem for + // real-time rendering if it happens too often + if (render_time > slot->frame_duration) { + GST_DEBUG_OBJECT( + state->plugin, + "Render GL frame took too long: %" GST_TIME_FORMAT + ", frame-duration: %" GST_TIME_FORMAT ", pts: %" GST_TIME_FORMAT, + GST_TIME_ARGS(render_time), GST_TIME_ARGS(slot->frame_duration), + GST_TIME_ARGS(slot->pts)); + } + + // copy params to locals vars to release the slot + GstBuffer *audio_buffer = slot->in_audio; + const GstClockTime frame_duration = slot->frame_duration; + const GstClockTime pts = slot->pts; +#ifdef RB_DROP_LATE_FRAMES + const GstClockTime latency = slot->latency; + const GstClockTime running_time = slot->running_time; +#endif + + // copy results to locals vars to release the slot + GstBuffer *outbuf = slot->out_buf; + const gboolean gl_result = slot->gl_result; + + // Lock and reset slot data + g_mutex_lock(&state->slot_lock); + + // signal render complete + g_cond_signal(&state->render_complete_cond); + + slot->in_audio = NULL; + slot->state = RB_EMPTY; + slot->out_buf = NULL; + slot->gl_result = FALSE; + + // let queuing know that a slot is available + g_cond_signal(&state->slot_available_cond); + g_mutex_unlock(&state->slot_lock); + + // populate timestamps after rendering so they can't be changed by accident + GST_BUFFER_PTS(outbuf) = pts; + GST_BUFFER_DTS(outbuf) = pts; + GST_BUFFER_DURATION(outbuf) = frame_duration; + + if (gst_buffer_get_size(outbuf) == 0) { + GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); + rb_queue_gl_buffer_cleanup(state, outbuf); + outbuf = NULL; + } else { + + // we got a rendered buffer, perform rendering loop QoS +#ifdef RB_DROP_LATE_FRAMES + gboolean dropped = FALSE; + + if (state->is_pipeline_realtime) { + dropped = rb_is_render_too_late(GST_ELEMENT(state->plugin), latency, + running_time); + } + if (!dropped) { +#endif + + // push buffer downstream + const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); + if (ret != GST_FLOW_OK) { + GST_WARNING("Failed to push buffer to pad"); + } + +#ifdef RB_DROP_LATE_FRAMES + } else { + rb_queue_gl_buffer_cleanup(state, outbuf); + outbuf = NULL; + } +#endif + + // process rendering fps QoS + if (state->qos_enabled) { + rb_handle_adaptive_fps_ema(state, render_time, frame_duration); + } + } + + gst_buffer_unref(audio_buffer); + + if (!gl_result) { + GST_WARNING_OBJECT( + state->plugin, + "Failed to render buffer, gl rendering returned error"); + } + + g_mutex_lock(&state->slot_lock); + } + + g_mutex_unlock(&state->slot_lock); + + return NULL; +} + +void rb_start_render_thread(RBRenderBuffer *state, GstGLContext *gl_context, + GstPad *src_pad) { + state->gl_context = gl_context; + state->src_pad = src_pad; + state->running = TRUE; + state->render_thread = + g_thread_new("rb-render-thread", rb_render_thread_func, state); + state->cleanup_thread = + g_thread_new("rb-cleanup-thread", cleanup_thread_func, state); + + GST_INFO_OBJECT(state->plugin, "Started render buffer"); +} + +void rb_stop_render_thread(RBRenderBuffer *state) { + + g_mutex_lock(&state->slot_lock); + state->running = FALSE; + + g_cond_broadcast(&state->render_queued_cond); + g_mutex_unlock(&state->slot_lock); + + g_thread_join(state->render_thread); + g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); + g_thread_join(state->cleanup_thread); + state->gl_context = NULL; + state->src_pad = NULL; + GST_INFO_OBJECT(state->plugin, "Stopped render buffer"); +} + +void rb_set_caps_frame_duration(RBRenderBuffer *state, + const GstClockTime caps_frame_duration) { + g_mutex_lock(&state->slot_lock); + state->caps_frame_duration = caps_frame_duration; + g_mutex_unlock(&state->slot_lock); +} diff --git a/src/renderbuffer.h b/src/renderbuffer.h new file mode 100644 index 0000000..da5ae0d --- /dev/null +++ b/src/renderbuffer.h @@ -0,0 +1,430 @@ +/* + * A ring buffer based render buffer to allow async offloading of + * rendering tasks from the plugin chain function. The ring buffer consists of + * a limited number of rendering slots. + * + * The buffer provides queueing for audio buffers to be rendered to video + * frames, supporting real-time and offline rendering. It uses a + * bound-wait-on-full approach to avoid dropping frames when rendering duration + * exceeds the frame duration of the current fps: + * + * - In case a free slot is available queue immediately and return. + * + * - In case the next available (not rendering) slot is scheduled (end of the + * ring + 1): + * + * - (real-time pipelines only) Wait for defined time for a slot to become + * available, this wait may not exceed the current fps frame duration, + * otherwise the plugin loses audio sync and fails. + * + * - (real-time pipelines only) In case the max wait deadline is met, + * and the next buffer still hasn't been picked up, it is overridden + * with the current frame (evicted), meaning the previous frame is being + * dropped as it is too late. + * + * - (offline pipelines only) Always wait until the next slot is free. + * + * For real-time pipelines only: + * + * - If the render duration exceeds the fps *sometimes*, subsequent + * faster-than-real-time rendered frames (if any) compensate for the small + * lag, frames are dropped or eventually QoS events from the downstream + * sink will re-sync with the pipeline clock. + * + * - If the render duration exceeds the fps *most of the time*, an Exponential + * Moving Average based algorithm instructs the plugin to reduce fps. + */ + +#ifndef __RENDERBUFFER_H__ +#define __RENDERBUFFER_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +/** + * Number of render slots that are used by the ring buffer. + * 2 is the ideal size and there should be no reason to change it: + * One slot for the gl thread to render the current frame while another slot is + * available for queuing the next audio buffer to render. + * Note: GstSegments won't be handled correctly currently if the number of slots + * is increased. + * + * Valid values: + * 1 - Wait for previous render to complete before scheduling. + * 2 - Render and schedule one item and at the same time. + */ +#ifndef NUM_RENDER_SLOTS +#define NUM_RENDER_SLOTS 2 +#endif + +/** + * Callback function pointer type for triggering a dynamic fps change. + */ +typedef void (*RBAdjustFpsFunc)(gpointer user_data, guint64 frame_duration); + +/** + * Current usage state of a render slot. + */ +typedef enum { + /** + * Slot is not in use at all. + */ + RB_EMPTY, + + /** + * Ready to render, in_audio buffer is filled. + */ + RB_READY, + + /** + * Slot is currently being rendered. + */ + RB_BUSY +} RQSlotState; + +/** + * Result status of queuing a buffer for rendering. + */ +typedef enum { + /** + * Buffer has been queued. + */ + RB_SUCCESS, + + /** + * Queuing buffer evicted (overwrote) a previously queued buffer (frame drop). + */ + RB_EVICTED, + + /** + * Buffer could not be queued because the allowed wait could not be met. + */ + RB_TIMEOUT +} RBQueueResult; + +/** + * A render slot represents an item in the render buffer. It holds an audio + * input buffer used for a single frame, render context information like frame + * pts and duration, and an output buffer for the rendered video frame. + */ +typedef struct { + + // not re-assigned + // -------------------------------------------------------------- + + /** + * projectM plugin. + */ + GstObject *plugin; + + /** + * Callback to render to gl texture buffer. + */ + GstGLContextThreadFunc gl_fill_func; + + // input for rendering, updated by queuing for each frame + // -------------------------------------------------------------- + + /** + * Presentation timestamp for this video frame. + */ + GstClockTime pts; + + /** + * Duration for this video frame (current fps). + */ + GstClockTime frame_duration; + + /** + * Audio data to feed to projectM for this frame. + */ + GstBuffer *in_audio; + + /** + * Current pipeline latency. + */ + GstClockTime latency; + + /** + * Running time for this video frame. + */ + GstClockTime running_time; + + // output from rendering, updated by gl thread for each frame + // -------------------------------------------------------------- + + /** + * GL memory texture buffer for current frame. + */ + GstBuffer *out_buf; + + /** + * GL render result for current frame. + */ + gboolean gl_result; + + // frequently updated, more than once for each frame + // -------------------------------------------------------------- + + /** + * Usage state of this slot. + */ + RQSlotState state; + +} RBSlot; + +/** + * All render buffer data. + */ +typedef struct { + + // not re-assigned during render thread lifetime + // -------------------------------------------------------------- + + /** + * projectM plugin. No ownership. + */ + GstObject *plugin; + + /** + * Current gl context. No ownership. + */ + GstGLContext *gl_context; + + /** + * projectM plugin source pad. No ownership. + */ + GstPad *src_pad; + + /** + * Thread running the render loop. + */ + GThread *render_thread; + + /** + * Thread running the gl buffer clean-up loop, + * used to release dropped buffer from the gl thread. + */ + GThread *cleanup_thread; + + /** + * Queue to dispose of dropped gl buffers. + */ + GAsyncQueue *buffer_cleanup_queue; + + /** + * Callback function pointer to let the plugin know to change fps. + */ + RBAdjustFpsFunc adjust_fps_func; + + /** + * Lock for shared state between chain function and render thread. + */ + GMutex slot_lock; + + // concurrent access, protected by slot_lock + // -------------------------------------------------------------- + + /** + * Threads currently running. + */ + gboolean running; + + /** + * Condition to wait for a buffer to queued for rendering. + */ + GCond render_queued_cond; + + /** + * Condition for slots becoming available after rendering completed. + */ + GCond slot_available_cond; + + /** + * Condition for rendering completion. + */ + GCond render_complete_cond; + + /** + * Is current pipeline using a real-time clock. + */ + gboolean qos_enabled; + + /** + * Pipeline negotiated caps fps as frame duration. + */ + GstClockTime caps_frame_duration; + + /** + * Limit for max EMA fps changes as frame duration. Higher value = lower fps. + */ + GstClockTime max_frame_duration; + + /** + * Render ring buffer slots. + */ + RBSlot slots[NUM_RENDER_SLOTS]; + + // only used by the calling thread (chain function) + // -------------------------------------------------------------- + + /** + * Last index that data was inserted at (insertion pointer). + */ + gint last_insert_index; + + // only used by the render thread + // -------------------------------------------------------------- + + /** + * Last index that data was rendered from (read pointer). + */ + gint last_render_index; + + /** + * EMA frame counter. + */ + guint frame_counter; + + /** + * EMA running average. + */ + guint64 smoothed_render_time; + +} RBRenderBuffer; + +/** + * Call argument struct, input for queuing a frame for rendering. + */ +typedef struct { + + /** + * Render buffer to use. + */ + RBRenderBuffer *render_buffer; + + /** + * Max time to wait for queuing. + */ + GstClockTime max_wait; + + /** + * If TRUE, the queuing call will block until rendering completed. + */ + gboolean sync_rendering; + + /** + * Presentation timestamp for this video frame. + */ + GstClockTime pts; + + /** + * Duration for this video frame (current fps). + */ + GstClockTime frame_duration; + + /** + * Audio data to feed to projectM for this frame. + */ + GstBuffer *in_audio; + + /** + * Current pipeline latency. + */ + GstClockTime latency; + + /** + * Running time for this video frame. + */ + GstClockTime running_time; + +} RBQueueArgs; + +/** + * One time initialization for the given render buffer. + * + * @param state Render buffer to use. + * @param plugin Plugin using the render buffer. + * @param gl_fill_func GL rendering function callback. + * @param adjust_fps_func FPS adjustment function callback. + * @param max_frame_duration FPS adjustment lower limit. + * @param caps_frame_duration FPS requested by pipeline caps. + * @param is_qos_enabled Controls if render-time QoS is enabled (EMA). + */ +void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, + GstGLContextThreadFunc gl_fill_func, + RBAdjustFpsFunc adjust_fps_func, + GstClockTime max_frame_duration, + GstClockTime caps_frame_duration, + gboolean is_qos_enabled); + +/** + * Release resources for the given render buffer. + * + * @param state Render buffer to clean up. + */ +void rb_dispose_render_buffer(RBRenderBuffer *state); + +/** + * Queue an audio buffer for rendering. The queuing is guaranteed to return + * within the given max time budget. The buffer will be dropped if queuing is + * not possible within the given time budget. + * + * @param args Audio buffer and frame details for rendering. The render buffer + * does not take ownership of the given pointer. + */ +RBQueueResult rb_queue_render_job(RBQueueArgs *args); + +/** + * Queue an audio buffer for rendering. The queuing is guaranteed to return + * within the given max time budget. The buffer will be dropped if queuing is + * not possible within the given time budget. + * + * Convenience function that also handles queuing result by warning if frames + * are dropped. + * + * @param args Audio buffer and frame details for rendering. The render buffer + * does not take ownership of the given pointer. + */ +void rb_queue_render_job_warn(RBQueueArgs *args); + +/** + * Start render loop. + * + * @param state Render buffer to use. + * @param gl_context GL context to use for rendering. + * @param src_pad Source pad to push video buffers to. + */ +void rb_start_render_thread(RBRenderBuffer *state, GstGLContext *gl_context, + GstPad *src_pad); + +/** + * Stop render loop. Active threads will be joined before returning. + * + * @param state Render buffer to use. + */ +void rb_stop_render_thread(RBRenderBuffer *state); + +/** + * Update caps as they get negotiated by the pipeline. Thread safe. + * + * @param state Render buffer to update. + * @param caps_frame_duration Frame duration from pipeline caps. + */ +void rb_set_caps_frame_duration(RBRenderBuffer *state, + GstClockTime caps_frame_duration); + +/** + * Update clock source type for the pipeline. Thread safe. + * + * @param state Render buffer to update. + * @param is_qos_enabled TRUE if the pipeline is using a real-time clock. + */ +void rb_set_qos_enabled(RBRenderBuffer *state, gboolean is_qos_enabled); + +G_END_DECLS + +#endif // __RENDERBUFFER_H__ From a5258fbff2f543d713340f1d2ef4a14d7648ef70 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 6 Oct 2025 22:58:30 -0500 Subject: [PATCH 02/32] update docs, formatting --- README.md | 23 ++++++++++++----------- src/debug.h | 1 - src/gstglbaseaudiovisualizer.c | 6 +++--- src/gstglbaseaudiovisualizer.h | 4 ++-- src/gstpmaudiovisualizer.c | 2 +- src/gstprojectm.c | 5 ++--- src/gstprojectm.h | 9 +++++---- src/gstprojectmbase.c | 9 ++++----- src/gstprojectmcaps.h | 2 -- src/register.c | 4 ++-- src/renderbuffer.c | 5 +++++ src/renderbuffer.h | 29 +++++++++++++++++++---------- 12 files changed, 55 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 38d85d3..a19e1c1 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ gst-inspect projectm ### ⏱️ Timing and Synchronization -The plugin synchronizes rendering to the GStreamer pipeline clock using **audio PTS (presentation timestamp) as the leading reference**. +The plugin synchronizes rendering to the GStreamer pipeline clock using **audio presentation timestamp (PTS) as the leading reference**. Pipeline caps control the desired video framerate for rendering. The render loop is **push-based** to conform with GStreamer's pipeline timing concept, and to enable faster-than-real-time rendering. @@ -268,20 +268,21 @@ A **fixed number of audio samples is consumed per video frame**. **Example:** `735 samples per frame at 44.1 kHz = ~60 FPS.` -In real-time pipelines, frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with -pipeline caps fps. +Real-time pipelines only: Frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with +pipeline caps FPS. Video frame PTS offset is derived from the **first audio buffer PTS** or **segment event** plus accumulated samples to align with audio timing. -| Timing Source | Source | Applies to clock | Purpose | -|----------------------------------|--------------------|------------------|--------------------------------------------------------------------------------------------| -| Audio Timestamps | Audio Input | Always | Determine video timing and sync. | -| Sample Rate / Pipeline FPS | Audio Input / Caps | Always | Defines how many audio samples are used per frame and target FPS. | -| Segment Info | Segment Event | Always | Tracks running time and playback position. Used for PTS offsets. | -| QoS Feedback | QoS Event | Real-time | Skips outdated frames to reduce latency. | -| Render Frame Drop | Render Loop | Real-time | Drop frames that cannot be rendered in time. | -| Exponential Moving Average (EMA) | Render Loop | Real-time | Adjust plugin target fps when frame render time exceeds real-time budget most of the time. | +| Timing Source | Origin | Applies to clock | Purpose | +|----------------------------------|--------------------|------------------|---------------------------------------------------------------------------------------------------| +| Audio Timestamps | Audio Input | Always | Determine video timing and sync. | +| Sample Rate / Pipeline FPS | Audio Input / Caps | Always | Defines how many audio samples are used per frame and target FPS. | +| Segment Info | Segment Event | Always | Tracks running time and playback position. Used for PTS offsets. | +| QoS Feedback | QoS Event | Real-time | Skips outdated frames to reduce latency. | +| Render Frame Drop | Render Loop | Real-time | Drop frames that cannot be rendered in time. | +| Exponential Moving Average (EMA) | Render Loop | Real-time | Adjust plugin target FPS in case frame render time exceeds the real-time budget most of the time. | +| Latency Event | Render Loop | Real-time | Inform upstream of latency changes in case of adaptive FPS changes (EMA). | --- diff --git a/src/debug.h b/src/debug.h index 8096afd..bb67133 100644 --- a/src/debug.h +++ b/src/debug.h @@ -1,7 +1,6 @@ #ifndef __GST_PROJECTM_DEBUG_H__ #define __GST_PROJECTM_DEBUG_H__ -#include #include G_BEGIN_DECLS diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index df92909..2ce0a73 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -34,13 +34,13 @@ #include "config.h" #endif +#include +#include + #include "gstglbaseaudiovisualizer.h" #include "gstpmaudiovisualizer.h" #include "renderbuffer.h" -#include -#include - /** * SECTION:GstGLBaseAudioVisualizer * @short_description: #GstPMAudioVisualizer subclass for injecting OpenGL diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index a250c4e..e3126d7 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -32,9 +32,9 @@ #ifndef __GST_GL_BASE_AUDIO_VISUALIZER_H__ #define __GST_GL_BASE_AUDIO_VISUALIZER_H__ -#include "gstpmaudiovisualizer.h" #include -#include + +#include "gstpmaudiovisualizer.h" typedef struct _GstGLBaseAudioVisualizer GstGLBaseAudioVisualizer; typedef struct _GstGLBaseAudioVisualizerClass GstGLBaseAudioVisualizerClass; diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 0cae8b7..1b492e0 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -76,10 +76,10 @@ #include +#include #include #include "gstpmaudiovisualizer.h" -#include GST_DEBUG_CATEGORY_STATIC(pm_audio_visualizer_debug); #define GST_CAT_DEFAULT (pm_audio_visualizer_debug) diff --git a/src/gstprojectm.c b/src/gstprojectm.c index ff795c1..858f8b3 100644 --- a/src/gstprojectm.c +++ b/src/gstprojectm.c @@ -6,11 +6,10 @@ #include #endif -#include "gstprojectm.h" - -#include #include +#include "gstprojectm.h" + #include "debug.h" #include "gstglbaseaudiovisualizer.h" #include "gstprojectmcaps.h" diff --git a/src/gstprojectm.h b/src/gstprojectm.h index aa2a001..bf9239e 100644 --- a/src/gstprojectm.h +++ b/src/gstprojectm.h @@ -13,11 +13,12 @@ G_DECLARE_FINAL_TYPE(GstProjectM, gst_projectm, GST, PROJECTM, GstGLBaseAudioVisualizer) /* - * Main plug-in. Handles interactions with projectM. + * Main GstElement for this plug-in. Handles interactions with projectM. * Uses GstPMAudioVisualizer for handling audio-visualization (audio input, - * timing, video frame data). GstGLBaseAudioVisualizer extends - * GstPMAudioVisualizer to add gl context handling and is used by this plugin - * directly. GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. + * timing, buffer pool, chain function). GstGLBaseAudioVisualizer (video frame + * data, GL memory allocation, GL rendering) extends GstPMAudioVisualizer to add + * gl context handling and is used by this plugin directly. Hierarchy: + * GstProjectM -> GstGLBaseAudioVisualizer -> GstPMAudioVisualizer. */ struct _GstProjectM { GstGLBaseAudioVisualizer element; diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index 6c9fad2..c3a55c0 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -3,14 +3,12 @@ #include "config.h" #endif -#include "gstprojectmbase.h" - -#include "gstprojectmconfig.h" +#include #include "debug.h" #include "gstglbaseaudiovisualizer.h" - -#include +#include "gstprojectmbase.h" +#include "gstprojectmconfig.h" GST_DEBUG_CATEGORY_STATIC(gst_projectm_base_debug); #define GST_CAT_DEFAULT gst_projectm_base_debug @@ -105,6 +103,7 @@ static void gst_projectm_base_handle_preset_change(bool is_hard_cut, if (gst_debug_category_get_threshold(gst_projectm_base_debug) >= GST_LEVEL_INFO) { + char *name = projectm_playlist_item((projectm_playlist_handle)user_data, index); diff --git a/src/gstprojectmcaps.h b/src/gstprojectmcaps.h index 53237f7..1b22835 100644 --- a/src/gstprojectmcaps.h +++ b/src/gstprojectmcaps.h @@ -3,8 +3,6 @@ #include -#include "gstprojectm.h" - G_BEGIN_DECLS /** diff --git a/src/register.c b/src/register.c index 09f9238..3065cd6 100644 --- a/src/register.c +++ b/src/register.c @@ -3,11 +3,11 @@ #include "config.h" #endif +#include + #include "gstprojectm.h" #include "gstprojectmconfig.h" -#include - /* * This unit registers all gst elements from this plugin library to make them * available to GStreamer. diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 96c006d..bd75a4d 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -481,6 +481,11 @@ static gpointer rb_render_thread_func(gpointer user_data) { #if NUM_RENDER_SLOTS > 2 // wontfix: segment events would need to be handled for this check to // work right otherwise last_pts is not reset when the pts changes. + // if this is ever desired, each queued frame should have an + // incrementing id field to use for this check + + // check if next frame is already outdated, may happen if write + // pointer jumps over the read pointer. && (last_pts == 0 || slot->pts > last_pts) #endif ) { diff --git a/src/renderbuffer.h b/src/renderbuffer.h index da5ae0d..336c40a 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -1,14 +1,18 @@ /* - * A ring buffer based render buffer to allow async offloading of + * A ring buffer based render buffer to allow offloading of * rendering tasks from the plugin chain function. The ring buffer consists of * a limited number of rendering slots. * * The buffer provides queueing for audio buffers to be rendered to video - * frames, supporting real-time and offline rendering. It uses a + * frames, supporting real-time (async) and offline (sync) rendering. It uses a * bound-wait-on-full approach to avoid dropping frames when rendering duration * exceeds the frame duration of the current fps: * - * - In case a free slot is available queue immediately and return. + * - (offline pipelines only) In case a free slot is available queue immediately + * and wait for rendering to complete (sync rendering). + * + * - (real-time pipelines only) In case a free slot is available queue + * immediately and return (async rendering). * * - In case the next available (not rendering) slot is scheduled (end of the * ring + 1): @@ -22,7 +26,8 @@ * with the current frame (evicted), meaning the previous frame is being * dropped as it is too late. * - * - (offline pipelines only) Always wait until the next slot is free. + * - (offline pipelines only) Always wait until the next slot is free, and + * wait until rendering completed (sync rendering). * * For real-time pipelines only: * @@ -32,7 +37,8 @@ * sink will re-sync with the pipeline clock. * * - If the render duration exceeds the fps *most of the time*, an Exponential - * Moving Average based algorithm instructs the plugin to reduce fps. + * Moving Average (EMA) based algorithm instructs the plugin to reduce fps. + * EMA will also recover fps when render performance increases again. */ #ifndef __RENDERBUFFER_H__ @@ -48,9 +54,11 @@ G_BEGIN_DECLS * Number of render slots that are used by the ring buffer. * 2 is the ideal size and there should be no reason to change it: * One slot for the gl thread to render the current frame while another slot is - * available for queuing the next audio buffer to render. - * Note: GstSegments won't be handled correctly currently if the number of slots - * is increased. + * available for queuing the next audio buffer to render. Increasing the number + * of slots will just increase the potential for latency. + * Note: Increasing the number of slots >2 can be done but is not fully + * supported. GstSegments won't be handled correctly currently. See inline code + * comments. * * Valid values: * 1 - Wait for previous render to complete before scheduling. @@ -418,10 +426,11 @@ void rb_set_caps_frame_duration(RBRenderBuffer *state, GstClockTime caps_frame_duration); /** - * Update clock source type for the pipeline. Thread safe. + * Controls if real-time QoS is enabled. Thread safe. + * Should be enabled if the pipeline is using a real-time clock. * * @param state Render buffer to update. - * @param is_qos_enabled TRUE if the pipeline is using a real-time clock. + * @param is_qos_enabled TRUE real-time QoS is enabled. */ void rb_set_qos_enabled(RBRenderBuffer *state, gboolean is_qos_enabled); From 49e0b0dcaaaa63d26f60adc4125803a6dbeb605d Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 6 Oct 2025 23:09:11 -0500 Subject: [PATCH 03/32] use blocking rendering call for offline pipelines --- src/gstglbaseaudiovisualizer.c | 46 +++--- src/renderbuffer.c | 246 +++++++++++++++++++-------------- src/renderbuffer.h | 77 +++++++---- 3 files changed, 215 insertions(+), 154 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 2ce0a73..65447b3 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -56,7 +56,9 @@ * for initializing and cleaning up OpenGL resources. The `render` * virtual method of the GstPMAudioVisualizer is implemented to perform OpenGL * rendering. The implementer provides an implementation for fill_gl_memory to - * render directly to gl memory. + * render directly to gl memory. Rendering is performed blocking for + * offline rendering and asynchronously for real-time rendering. + * The plugin detects if the pipeline clock is a real-time clock. * * Typical plug-in call order for implementer-provided functions: * - gl_start (once) @@ -525,36 +527,38 @@ static GstFlowReturn gst_gl_base_audio_visualizer_fill( glav->priv->n_frames == 1)) goto eos; - // prepare args for queuing frame rendering - RBQueueArgs args; - args.render_buffer = &glav->priv->render_buffer; - args.in_audio = audio; - args.pts = pts; - args.frame_duration = frame_duration; - args.latency = bscope->latency; - args.running_time = running_time; - if (glav->priv->is_realtime == FALSE) { - // wait for each frame to complete - args.sync_rendering = TRUE; + g_rec_mutex_unlock(&glav->priv->context_lock); + + // offline rendering can be done synchronously, avoid queuing overhead + rb_render_blocking(&glav->priv->render_buffer, audio, pts, frame_duration, + bscope->latency, running_time); - // unlimited for offline rendering, frames will never be dropped by QoS. - args.max_wait = GST_CLOCK_TIME_NONE; + g_rec_mutex_lock(&glav->priv->context_lock); } else { - // fire and forget, mapping n samples per frame from upstream keeps us in - // sync - args.sync_rendering = FALSE; + // prepare args for queuing frame rendering + RBQueueArgs args; + args.render_buffer = &glav->priv->render_buffer; + args.in_audio = audio; + args.pts = pts; + args.frame_duration = frame_duration; + args.latency = bscope->latency; + args.running_time = running_time; // limit wait based on fps factor, make sure we never wait too long in order // to keep in sync args.max_wait = (GstClockTimeDiff)gst_util_uint64_scale_int( frame_duration, MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N, MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D); - } - // dispatch gst_gl_base_audio_visualizer_fill_gl to the gl render buffer, - // rendering is deferred. This may block for a while though. - rb_queue_render_job_warn(&args); + g_rec_mutex_unlock(&glav->priv->context_lock); + + // dispatch gst_gl_base_audio_visualizer_fill_gl to the gl render buffer, + // rendering is deferred. This may block for a while though. + rb_queue_render_job_log(&args); + + g_rec_mutex_lock(&glav->priv->context_lock); + } glav->priv->n_frames++; diff --git a/src/renderbuffer.c b/src/renderbuffer.c index bd75a4d..6b0b21b 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -64,16 +64,6 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; #define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D 10 #endif -/** - * Possible option to drop frames that are too late after rendering if they - * would be dropped by a downstream sink anyway. - * Experimental, tends to increase flicker in most cases. Disabled per default. - */ -// #define RB_DROP_LATE_FRAMES -#ifndef RB_DROP_LATE_FRAMES_TOLERANCE -#define RB_DROP_LATE_FRAMES_TOLERANCE (GST_MSECOND * 8) -#endif - /** * How much time has to be left of the time budget for scheduling before * entering wait. Tolerance to account for scheduling overhead etc. to guarantee @@ -203,7 +193,6 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, g_mutex_init(&state->slot_lock); g_cond_init(&state->slot_available_cond); g_cond_init(&state->render_queued_cond); - g_cond_init(&state->render_complete_cond); state->buffer_cleanup_queue = g_async_queue_new(); for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { @@ -224,7 +213,6 @@ void rb_dispose_render_buffer(RBRenderBuffer *state) { g_async_queue_unref(state->buffer_cleanup_queue); g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); - g_cond_clear(&state->render_complete_cond); g_mutex_clear(&state->slot_lock); } @@ -314,12 +302,6 @@ RBQueueResult rb_queue_render_job(RBQueueArgs *args) { // signal render thread that there is something to do g_cond_signal(&state->render_queued_cond); - - if (args->sync_rendering) { - // block until rendering completed, if requested - g_cond_wait(&state->render_complete_cond, &state->slot_lock); - } - g_mutex_unlock(&state->slot_lock); RBQueueResult result; @@ -331,7 +313,7 @@ RBQueueResult rb_queue_render_job(RBQueueArgs *args) { return result; } -void rb_queue_render_job_warn(RBQueueArgs *args) { +void rb_queue_render_job_log(RBQueueArgs *args) { RBRenderBuffer *state = args->render_buffer; const GstClockTime start_ts = gst_util_get_timestamp(); @@ -378,25 +360,14 @@ static GstClockTime rb_element_render_time(GstElement *element) { return now - base_time; } -/** - * Determine if it's likely too late push a buffer, as it would likely be - * dropped by a pipeline synchronized sink. - * - * @param element The plugin element. - * @param latency Pipeline latency. - * @param running_time Current buffer running time. - * @return TRUE in case the buffer is too late. - */ -static gboolean rb_is_render_too_late(GstElement *element, - const GstClockTime latency, - const GstClockTime running_time) { +gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, + const GstClockTime running_time, + const GstClockTime tolerance) { if (latency == GST_CLOCK_TIME_NONE) { return FALSE; } - const GstClockTime tolerance = RB_DROP_LATE_FRAMES_TOLERANCE; - const GstClockTime render_time = rb_element_render_time(element); // latest time to push this buffer for it to make it to sink in time @@ -447,6 +418,132 @@ static gpointer cleanup_thread_func(gpointer user_data) { return NULL; } +/** + * Render one frame for the given slot. + * + * @param state Render buffer to use. + * @param slot Prepared slot to render. + * + * @return Render duration. + */ +GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { + // measure rendering for QoS + GstClockTime render_start = gst_util_get_timestamp(); + + // Dispatch slot to GL thread + gst_gl_context_thread_add(state->gl_context, slot->gl_fill_func, slot); + + GstClockTime render_time = gst_util_get_timestamp() - render_start; + + // render took longer than the frame duration, this is a problem for + // real-time rendering if it happens too often + if (render_time > slot->frame_duration) { + GST_DEBUG_OBJECT( + state->plugin, + "Render GL frame took too long: %" GST_TIME_FORMAT + ", frame-duration: %" GST_TIME_FORMAT ", pts: %" GST_TIME_FORMAT, + GST_TIME_ARGS(render_time), GST_TIME_ARGS(slot->frame_duration), + GST_TIME_ARGS(slot->pts)); + } + + return render_time; +} + +/** + * Send a video buffer to the source pad downstream. + * Buffer is checked and timestamps are populated before sending. + * + * @param state Render buffer to use. + * @param outbuf Video buffer to send downstream (takes ownership). + * @param pts Frame PTS. + * @param frame_duration Frame duration. + * @param latency Current pipeline latency. + * @param running_time Frame running time. + * @return TRUE if the buffer was pushed successfully. + */ +GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, + GstClockTime pts, + GstClockTime frame_duration, + GstClockTime latency, + GstClockTime running_time) { + + if (gst_buffer_get_size(outbuf) == 0) { + GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); + rb_queue_gl_buffer_cleanup(state, outbuf); + } else { + + // populate timestamps after rendering so they can't be changed by accident + GST_BUFFER_PTS(outbuf) = pts; + GST_BUFFER_DTS(outbuf) = pts; + GST_BUFFER_DURATION(outbuf) = frame_duration; + + // we got a rendered buffer, perform rendering loop QoS + + // push buffer downstream + const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); + if (ret != GST_FLOW_OK) { + GST_WARNING("Failed to push buffer to pad"); + } + return ret; + } + + return GST_FLOW_OK; +} + +/** + * Reset buffer references, set slot state to RB_EMPTY and signal. + * + * @param state Render buffer to use. + * @param slot Slot to release. + */ +static void rb_release_slot(RBRenderBuffer *state, RBSlot *slot) { + // Lock and reset slot data + g_mutex_lock(&state->slot_lock); + + slot->in_audio = NULL; + slot->state = RB_EMPTY; + slot->out_buf = NULL; + slot->gl_result = FALSE; + + // let queuing know that a slot is available + g_cond_signal(&state->slot_available_cond); + g_mutex_unlock(&state->slot_lock); +} + +GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, + GstClockTime pts, GstClockTime frame_duration, + GstClockTime latency, + GstClockTime running_time) { + // Lock and reset slot data + g_mutex_lock(&state->slot_lock); + + RBSlot *slot = &state->slots[0]; + slot->in_audio = in_audio; + slot->state = RB_BUSY; + slot->out_buf = NULL; + slot->pts = pts; + slot->gl_result = FALSE; + slot->frame_duration = frame_duration; + slot->latency = latency; + slot->running_time = running_time; + + // perform rendering + rb_render_slot(state, slot); + + GstFlowReturn ret = rb_handle_send_buffer( + state, slot->out_buf, pts, frame_duration, latency, running_time); + + // reset slot + slot->in_audio = NULL; + slot->state = RB_EMPTY; + slot->out_buf = NULL; + slot->gl_result = FALSE; + + g_mutex_unlock(&state->slot_lock); + + return ret; +} + /** * Render thread main worker function. * @@ -467,7 +564,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { // first find a slot with data that's ready to render gboolean found_slot = FALSE; - RBSlot *slot; + RBSlot *slot = NULL; gint render_index; while (!found_slot) { render_index = (state->last_render_index + 1) % NUM_RENDER_SLOTS; @@ -501,11 +598,10 @@ static gpointer rb_render_thread_func(gpointer user_data) { // no slot means we're not running anymore if (found_slot == FALSE) { - g_cond_signal(&state->render_complete_cond); break; } - // update iteration maker + // update read maker state->last_render_index = render_index; #if NUM_RENDER_SLOTS > 2 last_pts = slot->pts; @@ -517,93 +613,33 @@ static gpointer rb_render_thread_func(gpointer user_data) { g_mutex_unlock(&state->slot_lock); - // measure rendering for QoS - GstClockTime render_start = gst_util_get_timestamp(); - - // Dispatch slot to GL thread - gst_gl_context_thread_add(state->gl_context, slot->gl_fill_func, slot); - - GstClockTime render_time = gst_util_get_timestamp() - render_start; - - // render took longer than the frame duration, this is a problem for - // real-time rendering if it happens too often - if (render_time > slot->frame_duration) { - GST_DEBUG_OBJECT( - state->plugin, - "Render GL frame took too long: %" GST_TIME_FORMAT - ", frame-duration: %" GST_TIME_FORMAT ", pts: %" GST_TIME_FORMAT, - GST_TIME_ARGS(render_time), GST_TIME_ARGS(slot->frame_duration), - GST_TIME_ARGS(slot->pts)); - } + // perform gl rendering + GstClockTime render_time = rb_render_slot(state, slot); // copy params to locals vars to release the slot GstBuffer *audio_buffer = slot->in_audio; const GstClockTime frame_duration = slot->frame_duration; const GstClockTime pts = slot->pts; -#ifdef RB_DROP_LATE_FRAMES const GstClockTime latency = slot->latency; const GstClockTime running_time = slot->running_time; -#endif // copy results to locals vars to release the slot GstBuffer *outbuf = slot->out_buf; const gboolean gl_result = slot->gl_result; - // Lock and reset slot data - g_mutex_lock(&state->slot_lock); - - // signal render complete - g_cond_signal(&state->render_complete_cond); - - slot->in_audio = NULL; - slot->state = RB_EMPTY; - slot->out_buf = NULL; - slot->gl_result = FALSE; - - // let queuing know that a slot is available - g_cond_signal(&state->slot_available_cond); - g_mutex_unlock(&state->slot_lock); - - // populate timestamps after rendering so they can't be changed by accident - GST_BUFFER_PTS(outbuf) = pts; - GST_BUFFER_DTS(outbuf) = pts; - GST_BUFFER_DURATION(outbuf) = frame_duration; + // release slot and signal + rb_release_slot(state, slot); - if (gst_buffer_get_size(outbuf) == 0) { - GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); - rb_queue_gl_buffer_cleanup(state, outbuf); - outbuf = NULL; - } else { - - // we got a rendered buffer, perform rendering loop QoS -#ifdef RB_DROP_LATE_FRAMES - gboolean dropped = FALSE; - - if (state->is_pipeline_realtime) { - dropped = rb_is_render_too_late(GST_ELEMENT(state->plugin), latency, - running_time); - } - if (!dropped) { -#endif - - // push buffer downstream - const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); - if (ret != GST_FLOW_OK) { - GST_WARNING("Failed to push buffer to pad"); - } - -#ifdef RB_DROP_LATE_FRAMES - } else { - rb_queue_gl_buffer_cleanup(state, outbuf); - outbuf = NULL; - } -#endif + // send out buffer downstream + if (rb_handle_send_buffer(state, outbuf, pts, frame_duration, latency, + running_time) == GST_FLOW_OK) { - // process rendering fps QoS + // process rendering fps QoS in case frame was pushed if (state->qos_enabled) { rb_handle_adaptive_fps_ema(state, render_time, frame_duration); } } + outbuf = NULL; gst_buffer_unref(audio_buffer); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 336c40a..5a5a40b 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -3,33 +3,32 @@ * rendering tasks from the plugin chain function. The ring buffer consists of * a limited number of rendering slots. * - * The buffer provides queueing for audio buffers to be rendered to video - * frames, supporting real-time (async) and offline (sync) rendering. It uses a - * bound-wait-on-full approach to avoid dropping frames when rendering duration - * exceeds the frame duration of the current fps: + * For offline pipelines only: + * + * - A blocking call is used for rendering, bypasses queuing. + * + * For real-time pipelines only: * - * - (offline pipelines only) In case a free slot is available queue immediately - * and wait for rendering to complete (sync rendering). + * The buffer provides queueing for audio buffers to be rendered to video + * frames, It uses a bound-wait-on-full approach to avoid dropping frames when + * rendering duration exceeds the frame duration of the current fps: * - * - (real-time pipelines only) In case a free slot is available queue + * - In case a free slot is available queue * immediately and return (async rendering). * * - In case the next available (not rendering) slot is scheduled (end of the * ring + 1): * - * - (real-time pipelines only) Wait for defined time for a slot to become + * - Wait for defined time for a slot to become * available, this wait may not exceed the current fps frame duration, * otherwise the plugin loses audio sync and fails. * - * - (real-time pipelines only) In case the max wait deadline is met, + * - In case the max wait deadline is met, * and the next buffer still hasn't been picked up, it is overridden * with the current frame (evicted), meaning the previous frame is being * dropped as it is too late. * - * - (offline pipelines only) Always wait until the next slot is free, and - * wait until rendering completed (sync rendering). - * - * For real-time pipelines only: + * --- * * - If the render duration exceeds the fps *sometimes*, subsequent * faster-than-real-time rendered frames (if any) compensate for the small @@ -251,11 +250,6 @@ typedef struct { */ GCond slot_available_cond; - /** - * Condition for rendering completion. - */ - GCond render_complete_cond; - /** * Is current pipeline using a real-time clock. */ @@ -319,11 +313,6 @@ typedef struct { */ GstClockTime max_wait; - /** - * If TRUE, the queuing call will block until rendering completed. - */ - gboolean sync_rendering; - /** * Presentation timestamp for this video frame. */ @@ -382,7 +371,8 @@ void rb_dispose_render_buffer(RBRenderBuffer *state); * not possible within the given time budget. * * @param args Audio buffer and frame details for rendering. The render buffer - * does not take ownership of the given pointer. + * does not take ownership of the given pointer. The given audio buffer is + * copied. */ RBQueueResult rb_queue_render_job(RBQueueArgs *args); @@ -391,13 +381,44 @@ RBQueueResult rb_queue_render_job(RBQueueArgs *args); * within the given max time budget. The buffer will be dropped if queuing is * not possible within the given time budget. * - * Convenience function that also handles queuing result by warning if frames - * are dropped. + * Convenience function that also handles queuing result by logging if frames + * are dropped (DEBUG level). * * @param args Audio buffer and frame details for rendering. The render buffer - * does not take ownership of the given pointer. + * does not take ownership of the given pointer. The given audio buffer is + * copied. + */ +void rb_queue_render_job_log(RBQueueArgs *args); + +/** + * Render one frame synchronously. Using synchronous rendering is exclusive, + * queuing may not be used with the same render buffer at the same time. + * + * @param state Render buffer to use. + * @param pts Frame PTS. + * @param frame_duration Frame duration. + * @param latency Current pipeline latency. + * @param running_time Frame running time. + * @return The downstream push result. + */ +GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, + GstClockTime pts, GstClockTime frame_duration, + GstClockTime latency, + GstClockTime running_time); + +/** + * Determine if it's likely too late push a buffer, as it would likely be + * dropped by a pipeline synchronized sink. + * + * @param element The plugin element. + * @param latency Pipeline latency. + * @param running_time Current buffer running time. + * @param tolerance Tolerance to account for scheduling overhead. + * @return TRUE in case the buffer is too late. */ -void rb_queue_render_job_warn(RBQueueArgs *args); +static gboolean rb_is_render_too_late(GstElement *element, GstClockTime latency, + GstClockTime running_time, + GstClockTime tolerance); /** * Start render loop. From 7d7955a8ca42695dda0d0b0d2e394b5c9bf0cadc Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Tue, 7 Oct 2025 00:57:05 -0500 Subject: [PATCH 04/32] remove unused parameters --- src/gstglbaseaudiovisualizer.c | 24 +++++++++++------------ src/gstpmaudiovisualizer.c | 17 +++++++---------- src/gstpmaudiovisualizer.h | 3 +-- src/renderbuffer.c | 27 +++++++------------------- src/renderbuffer.h | 35 +++++----------------------------- 5 files changed, 31 insertions(+), 75 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 65447b3..6ca8b65 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -134,9 +134,10 @@ gst_gl_base_audio_visualizer_change_state(GstElement *element, * Renders a video frame using gl, impl for parent class * GstPMAudioVisualizerClass. */ -static GstFlowReturn gst_gl_base_audio_visualizer_parent_render( - GstPMAudioVisualizer *bscope, GstBuffer *audio, GstClockTime pts, - GstClockTime running_time, guint64 frame_duration); +static GstFlowReturn +gst_gl_base_audio_visualizer_parent_render(GstPMAudioVisualizer *bscope, + GstBuffer *audio, GstClockTime pts, + guint64 frame_duration); /** * Internal utility for resetting state on start \ @@ -515,8 +516,7 @@ static void gst_gl_base_audio_visualizer_fill_gl(GstGLContext *context, static GstFlowReturn gst_gl_base_audio_visualizer_fill( GstPMAudioVisualizer *bscope, GstGLBaseAudioVisualizer *glav, - GstBuffer *audio, GstClockTime pts, GstClockTime running_time, - guint64 frame_duration) { + GstBuffer *audio, GstClockTime pts, guint64 frame_duration) { g_rec_mutex_lock(&glav->priv->context_lock); if (G_UNLIKELY(!glav->context)) @@ -531,8 +531,7 @@ static GstFlowReturn gst_gl_base_audio_visualizer_fill( g_rec_mutex_unlock(&glav->priv->context_lock); // offline rendering can be done synchronously, avoid queuing overhead - rb_render_blocking(&glav->priv->render_buffer, audio, pts, frame_duration, - bscope->latency, running_time); + rb_render_blocking(&glav->priv->render_buffer, audio, pts, frame_duration); g_rec_mutex_lock(&glav->priv->context_lock); } else { @@ -542,8 +541,6 @@ static GstFlowReturn gst_gl_base_audio_visualizer_fill( args.in_audio = audio; args.pts = pts; args.frame_duration = frame_duration; - args.latency = bscope->latency; - args.running_time = running_time; // limit wait based on fps factor, make sure we never wait too long in order // to keep in sync @@ -635,13 +632,14 @@ gst_gl_base_audio_visualizer_parent_setup(GstPMAudioVisualizer *pmav) { return glav_class->setup(glav); } -static GstFlowReturn gst_gl_base_audio_visualizer_parent_render( - GstPMAudioVisualizer *bscope, GstBuffer *audio, GstClockTime pts, - GstClockTime running_time, guint64 frame_duration) { +static GstFlowReturn +gst_gl_base_audio_visualizer_parent_render(GstPMAudioVisualizer *bscope, + GstBuffer *audio, GstClockTime pts, + guint64 frame_duration) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(bscope); return gst_gl_base_audio_visualizer_fill(bscope, glav, audio, pts, - running_time, frame_duration); + frame_duration); } static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav) { diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 1b492e0..7653675 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -410,10 +410,12 @@ static gboolean gst_pm_audio_visualizer_do_setup(GstPMAudioVisualizer *scope) { g_mutex_lock(&scope->priv->config_lock); - scope->priv->spf = gst_util_uint64_scale_int( + const guint spf = gst_util_uint64_scale_int( GST_AUDIO_INFO_RATE(&scope->ainfo), GST_VIDEO_INFO_FPS_D(&scope->vinfo), GST_VIDEO_INFO_FPS_N(&scope->vinfo)); - scope->req_spf = scope->priv->spf; + + scope->req_spf = spf; + scope->priv->spf = spf; g_mutex_unlock(&scope->priv->config_lock); @@ -430,8 +432,7 @@ static gboolean gst_pm_audio_visualizer_do_setup(GstPMAudioVisualizer *scope) { GST_AUDIO_INFO_CHANNELS(&scope->ainfo), GST_AUDIO_INFO_BPF(&scope->ainfo)); - GST_INFO_OBJECT(scope, "blocks: spf %u, req_spf %u", scope->priv->spf, - scope->req_spf); + GST_INFO_OBJECT(scope, "blocks: spf / req_spf %u", spf); g_mutex_lock(&scope->priv->config_lock); scope->priv->ready = TRUE; @@ -721,7 +722,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GstFlowReturn ret = GST_FLOW_OK; GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(parent); GstPMAudioVisualizerClass *klass; - GstClockTime ts, running_time, frame_duration; + GstClockTime ts, frame_duration; guint avail, sbpf; // databuf is a buffer holding one video frame worth of audio data used as // temp buffer for copying from the adapter only @@ -865,10 +866,6 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, /* map pts ts via segment for general use */ ts = gst_segment_to_stream_time(&scope->priv->segment, GST_FORMAT_TIME, ts); - /* get running time for passing through */ - running_time = - gst_segment_to_running_time(&scope->priv->segment, GST_FORMAT_TIME, ts); - ++scope->priv->processed; /* sync controlled properties */ @@ -888,7 +885,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, /* call class->render() vmethod */ g_mutex_unlock(&scope->priv->config_lock); - ret = klass->render(scope, inbuf, ts, running_time, frame_duration); + ret = klass->render(scope, inbuf, ts, frame_duration); if (ret != GST_FLOW_OK) { goto beach; } diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index a96e526..d815d36 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -123,8 +123,7 @@ struct _GstPMAudioVisualizerClass { * Virtual function for rendering a frame. */ GstFlowReturn (*render)(GstPMAudioVisualizer *scope, GstBuffer *audio, - GstClockTime pts, GstClockTime running_time, - guint64 frame_duration); + GstClockTime pts, guint64 frame_duration); /** * Virtual function for buffer pool allocation. diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 6b0b21b..cd4170b 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -201,8 +201,6 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, state->slots[i].gl_result = FALSE; state->slots[i].pts = GST_CLOCK_TIME_NONE; state->slots[i].frame_duration = 0; - state->slots[i].latency = GST_CLOCK_TIME_NONE; - state->slots[i].running_time = GST_CLOCK_TIME_NONE; state->slots[i].out_buf = NULL; state->slots[i].gl_fill_func = gl_fill_func; state->slots[i].in_audio = NULL; @@ -296,8 +294,6 @@ RBQueueResult rb_queue_render_job(RBQueueArgs *args) { slot->gl_result = FALSE; slot->pts = args->pts; slot->frame_duration = args->frame_duration; - slot->latency = args->latency; - slot->running_time = args->running_time; slot->in_audio = gst_buffer_copy_deep(args->in_audio); // signal render thread that there is something to do @@ -457,15 +453,11 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { * @param outbuf Video buffer to send downstream (takes ownership). * @param pts Frame PTS. * @param frame_duration Frame duration. - * @param latency Current pipeline latency. - * @param running_time Frame running time. * @return TRUE if the buffer was pushed successfully. */ GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, GstClockTime pts, - GstClockTime frame_duration, - GstClockTime latency, - GstClockTime running_time) { + GstClockTime frame_duration) { if (gst_buffer_get_size(outbuf) == 0) { GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); @@ -511,9 +503,8 @@ static void rb_release_slot(RBRenderBuffer *state, RBSlot *slot) { } GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, - GstClockTime pts, GstClockTime frame_duration, - GstClockTime latency, - GstClockTime running_time) { + GstClockTime pts, + GstClockTime frame_duration) { // Lock and reset slot data g_mutex_lock(&state->slot_lock); @@ -524,14 +515,12 @@ GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, slot->pts = pts; slot->gl_result = FALSE; slot->frame_duration = frame_duration; - slot->latency = latency; - slot->running_time = running_time; // perform rendering rb_render_slot(state, slot); - GstFlowReturn ret = rb_handle_send_buffer( - state, slot->out_buf, pts, frame_duration, latency, running_time); + GstFlowReturn ret = + rb_handle_send_buffer(state, slot->out_buf, pts, frame_duration); // reset slot slot->in_audio = NULL; @@ -620,8 +609,6 @@ static gpointer rb_render_thread_func(gpointer user_data) { GstBuffer *audio_buffer = slot->in_audio; const GstClockTime frame_duration = slot->frame_duration; const GstClockTime pts = slot->pts; - const GstClockTime latency = slot->latency; - const GstClockTime running_time = slot->running_time; // copy results to locals vars to release the slot GstBuffer *outbuf = slot->out_buf; @@ -631,8 +618,8 @@ static gpointer rb_render_thread_func(gpointer user_data) { rb_release_slot(state, slot); // send out buffer downstream - if (rb_handle_send_buffer(state, outbuf, pts, frame_duration, latency, - running_time) == GST_FLOW_OK) { + if (rb_handle_send_buffer(state, outbuf, pts, frame_duration) == + GST_FLOW_OK) { // process rendering fps QoS in case frame was pushed if (state->qos_enabled) { diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 5a5a40b..a4d8504 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -53,11 +53,10 @@ G_BEGIN_DECLS * Number of render slots that are used by the ring buffer. * 2 is the ideal size and there should be no reason to change it: * One slot for the gl thread to render the current frame while another slot is - * available for queuing the next audio buffer to render. Increasing the number - * of slots will just increase the potential for latency. - * Note: Increasing the number of slots >2 can be done but is not fully - * supported. GstSegments won't be handled correctly currently. See inline code - * comments. + * available for queuing the next audio buffer to render. + * + * Note: Increasing the number of slots >2 is not fully supported. + * GstSegments won't be handled correctly currently. See inline code comments. * * Valid values: * 1 - Wait for previous render to complete before scheduling. @@ -150,16 +149,6 @@ typedef struct { */ GstBuffer *in_audio; - /** - * Current pipeline latency. - */ - GstClockTime latency; - - /** - * Running time for this video frame. - */ - GstClockTime running_time; - // output from rendering, updated by gl thread for each frame // -------------------------------------------------------------- @@ -328,16 +317,6 @@ typedef struct { */ GstBuffer *in_audio; - /** - * Current pipeline latency. - */ - GstClockTime latency; - - /** - * Running time for this video frame. - */ - GstClockTime running_time; - } RBQueueArgs; /** @@ -397,14 +376,10 @@ void rb_queue_render_job_log(RBQueueArgs *args); * @param state Render buffer to use. * @param pts Frame PTS. * @param frame_duration Frame duration. - * @param latency Current pipeline latency. - * @param running_time Frame running time. * @return The downstream push result. */ GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, - GstClockTime pts, GstClockTime frame_duration, - GstClockTime latency, - GstClockTime running_time); + GstClockTime pts, GstClockTime frame_duration); /** * Determine if it's likely too late push a buffer, as it would likely be From 4bac1cccec2c0d4d8650d9114e8fc344375d3e35 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Tue, 7 Oct 2025 01:54:23 -0500 Subject: [PATCH 05/32] typos --- src/gstpmaudiovisualizer.c | 6 +----- src/renderbuffer.h | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 7653675..f1b9afa 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -42,10 +42,6 @@ * * The code has been modified to improve compatibility with projectM and OpenGL. * - * - New apis for implementer-provided memory allocation and video frame - * buffer mapping. Used by gl plugins for mapping video frames directly to gl - * memory. - * * - Main memory based video frame buffers have been removed. * * - Cpu based transition shaders have been removed. @@ -62,7 +58,7 @@ * * - Segment event propagation. * - * - All memory management and rendering is implementer-provided. + * - Memory management and rendering is implementer-provided. * * Typical plug-in call order for implementer-provided functions: * - decide_allocation (once) diff --git a/src/renderbuffer.h b/src/renderbuffer.h index a4d8504..3dcfdf4 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -10,7 +10,7 @@ * For real-time pipelines only: * * The buffer provides queueing for audio buffers to be rendered to video - * frames, It uses a bound-wait-on-full approach to avoid dropping frames when + * frames. It uses a bound-wait-on-full approach to avoid dropping frames when * rendering duration exceeds the frame duration of the current fps: * * - In case a free slot is available queue @@ -23,7 +23,7 @@ * available, this wait may not exceed the current fps frame duration, * otherwise the plugin loses audio sync and fails. * - * - In case the max wait deadline is met, + * - In case the max wait deadline is met, * and the next buffer still hasn't been picked up, it is overridden * with the current frame (evicted), meaning the previous frame is being * dropped as it is too late. From 4b7fd5a18f5b3040c59563552f2689e0dc78f1dd Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Tue, 7 Oct 2025 01:58:10 -0500 Subject: [PATCH 06/32] remove obsolete inlcudes --- src/gstglbaseaudiovisualizer.c | 1 - src/gstprojectmbase.h | 1 - src/renderbuffer.h | 1 - 3 files changed, 3 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 6ca8b65..a582705 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -34,7 +34,6 @@ #include "config.h" #endif -#include #include #include "gstglbaseaudiovisualizer.h" diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h index 952e571..439d408 100644 --- a/src/gstprojectmbase.h +++ b/src/gstprojectmbase.h @@ -7,7 +7,6 @@ #ifndef __GST_PROJECTM_BASE_H__ #define __GST_PROJECTM_BASE_H__ -#include #include #include #include diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 3dcfdf4..8141997 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -43,7 +43,6 @@ #ifndef __RENDERBUFFER_H__ #define __RENDERBUFFER_H__ -#include #include #include From 4ea811300737bd3a1d0cf4861fdb71f4fc1f6b56 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Tue, 7 Oct 2025 02:07:37 -0500 Subject: [PATCH 07/32] nit --- src/renderbuffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index cd4170b..285f270 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -543,7 +543,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { RBRenderBuffer *state = (RBRenderBuffer *)user_data; #if NUM_RENDER_SLOTS > 2 - GstClockTime last_pts = 0; + GstClockTime last_pts = GST_CLOCK_TIME_NONE; #endif // slot modifications are locked g_mutex_lock(&state->slot_lock); @@ -572,7 +572,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { // check if next frame is already outdated, may happen if write // pointer jumps over the read pointer. - && (last_pts == 0 || slot->pts > last_pts) + && (last_pts == GST_CLOCK_TIME_NONE || slot->pts > last_pts) #endif ) { found_slot = TRUE; From 60357f203aa114dc9120a494c8ea20bff2bcf2f0 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Tue, 7 Oct 2025 23:17:00 -0500 Subject: [PATCH 08/32] atomic access for running flag --- src/gstglbaseaudiovisualizer.c | 12 +++++------- src/renderbuffer.c | 24 +++++++++++------------- src/renderbuffer.h | 6 ++++-- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index a582705..45d9431 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -479,19 +479,16 @@ static void gst_gl_base_audio_visualizer_fill_gl(GstGLContext *context, // Check for GL sync meta GstGLSyncMeta *sync_meta = gst_buffer_get_gl_sync_meta(out_buf); - if (sync_meta) { - // wait until GPU is done using this buffer should not be needed - // gst_gl_sync_meta_wait(sync_meta, glav->context); - } - - // GstClockTime after_prepare = gst_util_get_timestamp(); + // if (sync_meta) { + // wait until GPU is done using this buffer should not be needed + // gst_gl_sync_meta_wait(sync_meta, glav->context); + // } // map output video frame to buffer outbuf with gl flags gst_video_frame_map(&out_video, &pmav->vinfo, out_buf, GST_MAP_WRITE | GST_MAP_GL | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); - // GstClockTime after_map = gst_util_get_timestamp(); GstAVRenderParams ds_rd; ds_rd.in_audio = render_slot->in_audio; ds_rd.mem = GST_GL_MEMORY_CAST(gst_buffer_peek_memory(out_buf, 0)); @@ -510,6 +507,7 @@ static void gst_gl_base_audio_visualizer_fill_gl(GstGLContext *context, gst_gl_sync_meta_set_sync_point(sync_meta, glav->context); render_slot->out_buf = out_buf; + // ownership transferred out_buf = NULL; } diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 285f270..ea2fba0 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -179,7 +179,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, state->gl_context = NULL; state->src_pad = NULL; state->render_thread = NULL; - state->running = FALSE; + g_atomic_int_set(&state->running, FALSE); state->qos_enabled = is_qos_enabled; state->caps_frame_duration = caps_frame_duration; @@ -402,7 +402,8 @@ static void gl_buffer_cleanup(GstGLContext *context, gpointer buf) { static gpointer cleanup_thread_func(gpointer user_data) { const RBRenderBuffer *state = (RBRenderBuffer *)user_data; - while (state->running) { + // consume gl buffers to dispatch to gl thread for cleanup + while (g_atomic_int_get(&state->running)) { gpointer item = g_async_queue_pop(state->buffer_cleanup_queue); @@ -469,8 +470,6 @@ GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, GST_BUFFER_DTS(outbuf) = pts; GST_BUFFER_DURATION(outbuf) = frame_duration; - // we got a rendered buffer, perform rendering loop QoS - // push buffer downstream const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); if (ret != GST_FLOW_OK) { @@ -546,15 +545,17 @@ static gpointer rb_render_thread_func(gpointer user_data) { GstClockTime last_pts = GST_CLOCK_TIME_NONE; #endif // slot modifications are locked - g_mutex_lock(&state->slot_lock); // start working on rendering frames until we shut down - while (state->running) { + while (g_atomic_int_get(&state->running)) { // first find a slot with data that's ready to render gboolean found_slot = FALSE; RBSlot *slot = NULL; gint render_index; + + g_mutex_lock(&state->slot_lock); + while (!found_slot) { render_index = (state->last_render_index + 1) % NUM_RENDER_SLOTS; @@ -579,7 +580,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { } else { // no data is ready, wait for a new audio buffer being pushed g_cond_wait(&state->render_queued_cond, &state->slot_lock); - if (state->running == FALSE) { + if (g_atomic_int_get(&state->running) == FALSE) { break; } } @@ -587,6 +588,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { // no slot means we're not running anymore if (found_slot == FALSE) { + g_mutex_unlock(&state->slot_lock); break; } @@ -635,12 +637,8 @@ static gpointer rb_render_thread_func(gpointer user_data) { state->plugin, "Failed to render buffer, gl rendering returned error"); } - - g_mutex_lock(&state->slot_lock); } - g_mutex_unlock(&state->slot_lock); - return NULL; } @@ -648,7 +646,7 @@ void rb_start_render_thread(RBRenderBuffer *state, GstGLContext *gl_context, GstPad *src_pad) { state->gl_context = gl_context; state->src_pad = src_pad; - state->running = TRUE; + g_atomic_int_set(&state->running, TRUE); state->render_thread = g_thread_new("rb-render-thread", rb_render_thread_func, state); state->cleanup_thread = @@ -660,7 +658,7 @@ void rb_start_render_thread(RBRenderBuffer *state, GstGLContext *gl_context, void rb_stop_render_thread(RBRenderBuffer *state) { g_mutex_lock(&state->slot_lock); - state->running = FALSE; + g_atomic_int_set(&state->running, FALSE); g_cond_broadcast(&state->render_queued_cond); g_mutex_unlock(&state->slot_lock); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 8141997..1ba533a 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -220,14 +220,16 @@ typedef struct { */ GMutex slot_lock; - // concurrent access, protected by slot_lock + // concurrent access, g_atomic // -------------------------------------------------------------- /** - * Threads currently running. + * TRUE if render thread is currently running. */ gboolean running; + // concurrent access, protected by slot_lock + // -------------------------------------------------------------- /** * Condition to wait for a buffer to queued for rendering. */ From a030fffdcd448e99924f58c648fa522da1a387e7 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Thu, 9 Oct 2025 02:31:29 -0500 Subject: [PATCH 09/32] remove example for now --- CMakeLists.txt | 2 - example/CMakeLists.txt | 30 ----- example/dynpads.c | 238 ------------------------------------- src/gstpmaudiovisualizer.c | 6 +- src/gstpmaudiovisualizer.h | 17 +-- 5 files changed, 13 insertions(+), 280 deletions(-) delete mode 100644 example/CMakeLists.txt delete mode 100644 example/dynpads.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ff14d2..7303333 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,5 +77,3 @@ target_link_libraries(gstprojectm ${GLIB2_LIBRARIES} ${GLIB2_GOBJECT_LIBRARIES} ) - -add_subdirectory(example) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt deleted file mode 100644 index f502740..0000000 --- a/example/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ - -add_executable(dyn-pads-example - dynpads.c -) - -target_include_directories(dyn-pads-example - PUBLIC - ${GSTREAMER_INCLUDE_DIRS} - ${GSTREAMER_BASE_INCLUDE_DIRS} - ${GSTREAMER_AUDIO_INCLUDE_DIRS} - ${GSTREAMER_GL_INCLUDE_DIRS} - ${GLIB2_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} -) - -target_link_libraries(dyn-pads-example - PRIVATE - libprojectM::projectM - libprojectM::playlist - PUBLIC - ${GSTREAMER_LIBRARIES} - ${GSTREAMER_BASE_LIBRARIES} - ${GSTREAMER_AUDIO_LIBRARIES} - ${GSTREAMER_VIDEO_LIBRARIES} - ${GSTREAMER_GL_LIBRARIES} - ${GSTREAMER_PBUTILS_LIBRARIES} - ${GLIB2_LIBRARIES} - ${GLIB2_GOBJECT_LIBRARIES} - gstprojectm -) diff --git a/example/dynpads.c b/example/dynpads.c deleted file mode 100644 index b388c5e..0000000 --- a/example/dynpads.c +++ /dev/null @@ -1,238 +0,0 @@ -#include - -#include - -/** - * Example for a "pad added" signal callback handler for handling gst - * demuxer-like elements. - * - * @param element Callback param for the gst element receiving the event. - * @param new_pad The pad being added. - * @param data The gst element adding the pad (e.g. demuxer). - */ -static void on_pad_added(GstElement *element, GstPad *new_pad, gpointer data) { - - GstPad *sink_pad; - GstElement *downstream_element = GST_ELEMENT(data); - GstPadLinkReturn ret; - GstCaps *new_pad_caps = NULL; - GstStructure *new_pad_struct = NULL; - const gchar *new_pad_type = NULL; - - g_print("Received new pad '%s' from '%s':\n", GST_PAD_NAME(new_pad), - GST_ELEMENT_NAME(element)); - - /* Check the new pad's capabilities to determine its media type */ - new_pad_caps = gst_pad_get_current_caps(new_pad); - new_pad_struct = gst_caps_get_structure(new_pad_caps, 0); - new_pad_type = gst_structure_get_name(new_pad_struct); - - /* Get the sink pad from the downstream element (either audio or video queue) - */ - if (g_str_has_prefix(new_pad_type, "video/x-raw")) { - sink_pad = gst_element_get_static_pad(downstream_element, "sink"); - } else if (g_str_has_prefix(new_pad_type, "audio/x-raw")) { - sink_pad = gst_element_get_static_pad(downstream_element, "sink"); - } else { - g_print(" It has type '%s', which we don't handle. Ignoring.\n", - new_pad_type); - goto exit; - } - - /* Check if the pads are already linked */ - if (gst_pad_is_linked(sink_pad)) { - g_print(" We already linked pad %s. Ignoring.\n", GST_PAD_NAME(new_pad)); - goto exit; - } - - /* Link the new pad to the sink pad */ - ret = gst_pad_link(new_pad, sink_pad); - if (GST_PAD_LINK_FAILED(ret)) { - g_print(" Type is '%s' but link failed.\n", new_pad_type); - } else { - g_print(" Link succeeded (type '%s').\n", new_pad_type); - } - -exit: - /* Clean up */ - if (new_pad_caps != NULL) - gst_caps_unref(new_pad_caps); - - if (sink_pad != NULL) - gst_object_unref(sink_pad); -} - -/** - * Main function to build and run the pipeline to consume a live audio stream - * and render projectM to an OpenGL window in real-time. - * - * souphttpsrc location=... is-live=true ! queue ! decodebin ! audioconvert ! - * "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! - * projectm preset=... preset-duration=... mesh-size=48,32 texture-dir=... ! - * video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! queue - * leaky=downstream max-size-buffers=1 ! glimagesink sync=true - */ -int main(int argc, char *argv[]) { - GstElement *source, *demuxer, *queue, *audioconvert, *audio_capsfilter, - *identity, *projectm_plugin, *video_capsfilter, *sync_queue, *sink; - GstBus *bus; - GstElement *pipeline; - - gst_init(&argc, &argv); - - // make audio caps - GstCaps *audio_caps = - gst_caps_new_simple("audio/x-raw", "format", G_TYPE_STRING, "S16LE", - "rate", G_TYPE_INT, 44100, "channels", G_TYPE_INT, 2, - "layout", G_TYPE_STRING, "interleaved", NULL); - - // make video caps - // todo: adjust caps as desired, keep in mind the hardware needs to be able to - // keep up in order for this plugin to work flawlessly. - GstCaps *video_caps = gst_caps_new_simple( - "video/x-raw", "format", G_TYPE_STRING, "RGBA", "width", G_TYPE_INT, 1920, - "height", G_TYPE_INT, 1080, "framerate", GST_TYPE_FRACTION, 60, 1, NULL); - - // Create the GL memory feature set. - GstCapsFeatures *features = - gst_caps_features_new_single(GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - - // Add the GL memory feature set to the structure. - gst_caps_set_features(video_caps, 0, features); - - // Create pipeline elements - source = gst_element_factory_make("souphttpsrc", "source"); - g_object_set(source, - // todo: configure your stream here.. - "location", "http://your-stream-url", "is-live", TRUE, NULL); - - // basic stream buffering - queue = gst_element_factory_make("queue", "queue"); - - // decodebin to decode the stream audio format - demuxer = gst_element_factory_make("decodebin", "demuxer"); - g_object_set(G_OBJECT(demuxer), "max-size-time", "100000000", NULL); - - // convert the audio stream to something we can understand (if needed) - audioconvert = gst_element_factory_make("audioconvert", "audioconvert"); - - // tell pipeline which audio format we need - audio_capsfilter = gst_element_factory_make("capsfilter", "audio_capsfilter"); - g_object_set(G_OBJECT(audio_capsfilter), "caps", audio_caps, NULL); - - // create an identity element to provide a sream clock, since we won't get one - // from souphttpsrc - identity = gst_element_factory_make("identity", "identity"); - g_object_set(G_OBJECT(identity), "single-segment", TRUE, "sync", TRUE, NULL); - - // configure projectM plugin - projectm_plugin = gst_element_factory_make("projectm", "projectm"); - - // todo: configure your settings here.. - g_object_set(G_OBJECT(projectm_plugin), "preset-duration", 10.0, - //"preset", "/your/presets/directory", - "mesh-size", "48,32", - //"texture-dir", "/your/presets-milkdrop-texture-pack-directory", - NULL); - - // set video caps we want - video_capsfilter = gst_element_factory_make("capsfilter", "video_capsfilter"); - g_object_set(G_OBJECT(video_capsfilter), "caps", video_caps, NULL); - - // optional: create a queue in front of the glimagesink to throw out buffers - // that piling up in front of rendering just keep the latest one, the others - // will most likely be late - sync_queue = gst_element_factory_make("queue", "sync_queue"); - // 0 (no): The default behavior. The queue is not leaky and will block when - // full. 1 (upstream): The queue drops new incoming buffers when it is full. - // 2 (downstream): The queue drops the oldest buffers in the queue when it is - // full. - g_object_set(G_OBJECT(sync_queue), "leaky", 2, "max-size-buffers", 1, NULL); - - // create sink for real-time rendering (synced to the pipeline clock) - sink = gst_element_factory_make("glimagesink", "sink"); - g_object_set(G_OBJECT(sink), "sync", TRUE, NULL); - - pipeline = gst_pipeline_new("test-pipeline"); - - if (!pipeline || !source || !demuxer || !queue || !projectm_plugin || - !video_capsfilter || !sync_queue || !sink) { - g_printerr("One or more elements could not be created. Exiting.\n"); - return -1; - } - - /* Set up the pipeline */ - gst_bin_add_many(GST_BIN(pipeline), source, queue, demuxer, audioconvert, - audio_capsfilter, identity, projectm_plugin, - video_capsfilter, sync_queue, sink, NULL); - - /* Link the elements (but not the demuxer's dynamic pad yet) */ - if (!gst_element_link(source, queue)) { - g_printerr("Elements could not be linked (source to queue). Exiting.\n"); - return -1; - } - if (!gst_element_link(queue, demuxer)) { - g_printerr( - "Elements could not be linked (queue, audioconvert). Exiting.\n"); - return -1; - } - /* not yet! - if (!gst_element_link(demuxer, audioconvert)) { - g_printerr("Elements could not be linked (demuxer, queue). Exiting.\n"); - return -1; - } - */ - if (!gst_element_link(audioconvert, audio_capsfilter)) { - g_printerr("Elements could not be linked (audioconvert to " - "audio_capsfilter). Exiting.\n"); - return -1; - } - if (!gst_element_link(audio_capsfilter, identity)) { - g_printerr("Elements could not be linked (audio_capsfilter to identity). " - "Exiting.\n"); - return -1; - } - if (!gst_element_link(identity, projectm_plugin)) { - g_printerr("Elements could not be linked (identity to projectm_plugin). " - "Exiting.\n"); - return -1; - } - if (!gst_element_link(projectm_plugin, video_capsfilter)) { - g_printerr("Elements could not be linked (projectm_plugin to capsfilter). " - "Exiting.\n"); - return -1; - } - if (!gst_element_link(video_capsfilter, sync_queue)) { - g_printerr("Elements could not be linked (video_capsfilter to sync_queue). " - "Exiting.\n"); - return -1; - } - if (!gst_element_link(sync_queue, sink)) { - g_printerr("Elements could not be linked (sync_queue to sink). Exiting.\n"); - return -1; - } - - gst_caps_unref(video_caps); - gst_caps_unref(audio_caps); - - /* Connect the "pad-added" signal */ - g_signal_connect(demuxer, "pad-added", G_CALLBACK(on_pad_added), - audioconvert); - - /* Set the pipeline to the PLAYING state */ - GMainLoop *loop = g_main_loop_new(NULL, FALSE); - gst_element_set_state(pipeline, GST_STATE_PLAYING); - g_main_loop_run(loop); - - /* Wait until error or EOS */ - bus = gst_element_get_bus(pipeline); - gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, - GST_MESSAGE_ERROR | GST_MESSAGE_EOS); - - /* Clean up */ - gst_element_set_state(pipeline, GST_STATE_NULL); - gst_object_unref(bus); - gst_object_unref(pipeline); - - return 0; -} diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index f1b9afa..cd17ba1 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -953,7 +953,7 @@ static gboolean gst_pm_audio_visualizer_src_event(GstPad *pad, scope->priv->proportion = proportion; if (diff > 0) { /* we're late, this is a good estimate for next displayable - * frame (see part-qos.txt) */ + * frame (see part-qos.txt) (skip all frames until this time) */ scope->priv->earliest_time = timestamp + MIN(diff * 2, GST_SECOND * 3) + scope->req_frame_duration; } else { @@ -1066,8 +1066,8 @@ static void gst_pm_audio_visualizer_send_latency_if_needed_unlocked( scope->priv->last_reported_latency = latency; g_mutex_unlock(&scope->priv->config_lock); gst_pad_push_event(scope->priv->sinkpad, gst_event_new_latency(latency)); - GST_INFO_OBJECT(scope, "Sent latency event to sink pad: %" GST_TIME_FORMAT, - GST_TIME_ARGS(latency)); + GST_DEBUG_OBJECT(scope, "Sent latency event to sink pad: %" GST_TIME_FORMAT, + GST_TIME_ARGS(latency)); g_mutex_lock(&scope->priv->config_lock); } } diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index d815d36..c4544b0 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -99,14 +99,17 @@ struct _GstPMAudioVisualizer { /** * GstPMAudioVisualizerClass: * @decide_allocation: buffer pool allocation - * @prepare_output_buffer: allocate a buffer for rendering a frame. - * @map_output_buffer: map video frame to memory buffer. * @render: render a frame from an audio buffer. - * @setup: called whenever the format changes. + * @setup: Called whenever the format changes, and sink and src caps are + * configured. + * @change_state: Cascades gst change state to the implementor. Parent is + * processed first. + * @segment_change: Cascades gst segment events to the implementor. Parent is + * processed first. * * Base class for audio visualizers, derived from gstreamer - * GstAudioVisualizerClass. This plugin handles rendering video frames with a - * fixed framerate from audio input samples. + * GstAudioVisualizerClass. This plugin consumes n audio input samples for each + * output video frame to keep audio and video in-sync. */ struct _GstPMAudioVisualizerClass { /*< private >*/ @@ -131,13 +134,13 @@ struct _GstPMAudioVisualizerClass { gboolean (*decide_allocation)(GstPMAudioVisualizer *scope, GstQuery *query); /** - * Virtual function to allow overridden change_state, cascading to GstElement. + * Virtual function to allow implementor to receive change_state events. */ GstStateChangeReturn (*change_state)(GstElement *element, GstStateChange transition); /** - * Virtual function to allow receiving segment change events. + * Virtual function allow implementor to receive segment change events. */ void (*segment_change)(GstPMAudioVisualizer *scope, GstSegment *segment); }; From 4c9db6c5f2bfe5e7408e213159fd5ca7c6f4ef84 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Fri, 10 Oct 2025 01:20:23 -0500 Subject: [PATCH 10/32] fix push timing for real-time rendering add is-live property --- src/gstglbaseaudiovisualizer.c | 143 +++++++++++++++++++++++---------- src/gstglbaseaudiovisualizer.h | 15 +++- src/gstpmaudiovisualizer.c | 22 ++--- src/gstprojectmbase.c | 80 ++++++++++++------ src/gstprojectmbase.h | 1 + src/renderbuffer.c | 120 +++++++++++++++++++++------ src/renderbuffer.h | 39 +++++---- 7 files changed, 295 insertions(+), 125 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 45d9431..74959e0 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -81,6 +81,7 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); #define DEFAULT_MIN_FPS_N 1 #define DEFAULT_MIN_FPS_D 1 +#define DEFAULT_IS_LIVE "auto" struct _GstGLBaseAudioVisualizerPrivate { GstGLContext *other_context; @@ -96,7 +97,7 @@ struct _GstGLBaseAudioVisualizerPrivate { }; /* Properties */ -enum { PROP_0, PROP_MIN_FPS_N, PROP_MIN_FPS_D }; +enum { PROP_0, PROP_MIN_FPS_N, PROP_MIN_FPS_D, PROP_IS_LIVE }; #define gst_gl_base_audio_visualizer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE( @@ -252,6 +253,15 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { "Specifies the denominator for the min fps (EMA)", 1, 1000, DEFAULT_MIN_FPS_D, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_IS_LIVE, + g_param_spec_string("is-live", "Is Live", + "Specifies if this element renders in real-time " + "(true) or as fast as possible for offline rendering " + "(false) or to auto-detect pipeline clock (auto)", + DEFAULT_IS_LIVE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } /** @@ -277,6 +287,7 @@ static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { glav->priv = gst_gl_base_audio_visualizer_get_instance_private(glav); glav->priv->gl_started = FALSE; glav->priv->fbo = NULL; + glav->is_live = GST_GL_BASE_AUDIO_VISUALIZER_AUTO; glav->priv->is_realtime = FALSE; glav->context = NULL; @@ -313,6 +324,17 @@ static void gst_gl_base_audio_visualizer_set_property(GObject *object, glav->min_fps_d = g_value_get_int(value); break; + case PROP_IS_LIVE: + const char *str = g_value_get_string(value); + if (strcasecmp("true", str) == 0) { + glav->is_live = GST_GL_BASE_AUDIO_VISUALIZER_REALTIME; + } else if (strcasecmp("false", str) == 0) { + glav->is_live = GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE; + } else { + glav->is_live = GST_GL_BASE_AUDIO_VISUALIZER_AUTO; + } + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -335,6 +357,16 @@ static void gst_gl_base_audio_visualizer_get_property(GObject *object, g_value_set_int(value, glav->min_fps_d); break; + case PROP_IS_LIVE: + if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_REALTIME) { + g_value_set_string(value, "true"); + } else if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE) { + g_value_set_string(value, "false"); + } else { + g_value_set_string(value, "auto"); + } + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -405,17 +437,18 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, gst_util_uint64_scale_int(GST_SECOND, GST_VIDEO_INFO_FPS_D(&pmav->vinfo), GST_VIDEO_INFO_FPS_N(&pmav->vinfo)); + // render loop QoS is disabled for offline rendering rb_init_render_buffer(&glav->priv->render_buffer, GST_OBJECT(glav), gst_gl_base_audio_visualizer_fill_gl, adjust_fps_callback, max_frame_duration, - caps_frame_duration, glav->priv->is_realtime); + caps_frame_duration, glav->priv->is_realtime, + glav->priv->is_realtime); // cascade gl start to implementor glav->priv->gl_started = glav_class->gl_start(glav); // get gl rendering going - rb_start_render_thread(&glav->priv->render_buffer, glav->context, - pmav->srcpad); + rb_start(&glav->priv->render_buffer, glav->context, pmav->srcpad); } static void @@ -432,7 +465,7 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, GST_OBJECT_NAME(glav)); // stop gl rendering first - rb_stop_render_thread(&glav->priv->render_buffer); + rb_stop(&glav->priv->render_buffer); // clean up implementor if (glav->priv->gl_started) { @@ -549,7 +582,7 @@ static GstFlowReturn gst_gl_base_audio_visualizer_fill( // dispatch gst_gl_base_audio_visualizer_fill_gl to the gl render buffer, // rendering is deferred. This may block for a while though. - rb_queue_render_job_log(&args); + rb_queue_render_task_log(&args); g_rec_mutex_lock(&glav->priv->context_lock); } @@ -574,55 +607,26 @@ eos: { } } -/** - * Find out if the pipeline is using a real-time clock. - * - * @param element GST element - * @return TRUE in case the element uses a system clock. - */ -static gboolean is_pipeline_realtime(GstElement *element) { - GstClock *clock = gst_element_get_clock(element); - gboolean is_realtime = FALSE; - - if (clock) { - // Compare to the system clock (used for real-time playback) - is_realtime = GST_IS_SYSTEM_CLOCK(clock); - gst_object_unref(clock); - } - - return is_realtime; -} - static gboolean gst_gl_base_audio_visualizer_parent_setup(GstPMAudioVisualizer *pmav) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(pmav); GstGLBaseAudioVisualizerClass *glav_class = GST_GL_BASE_AUDIO_VISUALIZER_GET_CLASS(pmav); - // configure QoS for the pipeline, disabled for offline rendering - g_rec_mutex_lock(&glav->priv->context_lock); - - gboolean is_realtime = is_pipeline_realtime(GST_ELEMENT(pmav)); - glav->priv->is_realtime = is_realtime; - - g_rec_mutex_unlock(&glav->priv->context_lock); - - // update render buffer config - rb_set_qos_enabled(&glav->priv->render_buffer, is_realtime); - GstClockTime caps_frame_duration = gst_util_uint64_scale_int(GST_SECOND, GST_VIDEO_INFO_FPS_D(&pmav->vinfo), GST_VIDEO_INFO_FPS_N(&pmav->vinfo)); + rb_set_caps_frame_duration(&glav->priv->render_buffer, caps_frame_duration); - GST_INFO_OBJECT( - glav, - "GL setup - render config: real-time: %s, caps-frame-duration: " - "%" GST_TIME_FORMAT - ", min-fps: %d/%d, min-fps-duration: %" GST_TIME_FORMAT, - is_realtime ? "true" : "false", GST_TIME_ARGS(caps_frame_duration), - glav->min_fps_n, glav->min_fps_d, - GST_TIME_ARGS(glav->priv->render_buffer.max_frame_duration)); + GST_INFO_OBJECT(glav, + "GL setup - render config: is-live: %s, caps-frame-duration: " + "%" GST_TIME_FORMAT + ", min-fps: %d/%d, min-fps-duration: %" GST_TIME_FORMAT, + glav->priv->is_realtime ? "true" : "false", + GST_TIME_ARGS(caps_frame_duration), glav->min_fps_n, + glav->min_fps_d, + GST_TIME_ARGS(glav->priv->render_buffer.max_frame_duration)); // cascade setup to the derived plugin after gl initialization has been // completed @@ -866,6 +870,42 @@ static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( return TRUE; } +/** + * Find pipeline clock and determine if it is a real-time clock. + * + * @param element Plugin element. + * + * @return TRUE if the pipeline clock is a real-time (system) clock. + */ +static gboolean is_pipeline_realtime(GstElement *element) { + GstClock *clock = NULL; + gboolean is_realtime = FALSE; + + // first check element's own clock, then start climbing the hierarchy + clock = gst_element_get_clock(element); + if (!clock) { + // traverse parents to find the pipeline and ask it for its clock + GstObject *parent = gst_object_get_parent(GST_OBJECT(element)); + while (parent && !GST_IS_PIPELINE(parent)) { + GstObject *next_parent = gst_object_get_parent(parent); + gst_object_unref(parent); + parent = next_parent; + } + if (parent && GST_IS_PIPELINE(parent)) { + GstPipeline *pipeline = GST_PIPELINE(parent); + clock = gst_pipeline_get_clock(pipeline); + gst_object_unref(parent); + } + } + + if (clock) { + is_realtime = GST_IS_SYSTEM_CLOCK(clock); + gst_object_unref(clock); + } + + return is_realtime; +} + static GstStateChangeReturn gst_gl_base_audio_visualizer_change_state(GstElement *element, GstStateChange transition) { @@ -884,6 +924,21 @@ gst_gl_base_audio_visualizer_change_state(GstElement *element, gst_clear_object(&glav->display); g_rec_mutex_unlock(&glav->priv->context_lock); break; + + case GST_STATE_CHANGE_READY_TO_PAUSED: + g_rec_mutex_lock(&glav->priv->context_lock); + // determine if we're using a real-time pipeline + if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE) { + glav->priv->is_realtime = FALSE; + } else if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_REALTIME) { + glav->priv->is_realtime = TRUE; + } else { + // auto detect + glav->priv->is_realtime = is_pipeline_realtime(element); + } + g_rec_mutex_unlock(&glav->priv->context_lock); + break; + default: break; } diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index e3126d7..4e58a8f 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -64,12 +64,18 @@ GType gst_gl_base_audio_visualizer_get_type(void); (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_GL_BASE_AUDIO_VISUALIZER, \ GstGLBaseAudioVisualizerClass)) +typedef enum { + GST_GL_BASE_AUDIO_VISUALIZER_REALTIME, + GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE, + GST_GL_BASE_AUDIO_VISUALIZER_AUTO +} GstGLBaseAudioVisualizerMode; + /** * GstGLBaseAudioVisualizer: * @display: the currently configured #GstGLDisplay * @context: the currently configured #GstGLContext * - * The parent instance type of a base GL Audio Visualizer. + * The parent instance type of base GL Audio Visualizer. */ struct _GstGLBaseAudioVisualizer { GstPMAudioVisualizer parent; @@ -88,6 +94,11 @@ struct _GstGLBaseAudioVisualizer { */ gint min_fps_d; + /** + * Operation mode property. + */ + GstGLBaseAudioVisualizerMode is_live; + /*< private >*/ gpointer _padding[GST_PADDING]; @@ -97,7 +108,7 @@ struct _GstGLBaseAudioVisualizer { /** * GstGLBaseAudioVisualizerClass: * @supported_gl_api: the logical-OR of #GstGLAPI's supported by this element - * @gl_start: called in the GL thread to setup the element GL state. + * @gl_start: called in the GL thread to set up the element GL state. * @gl_stop: called in the GL thread to clean up the element GL state. * @gl_render: called in the GL thread to fill the current video texture. * @setup: called when the format changes (delegate from diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index cd17ba1..35928c8 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -718,7 +718,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GstFlowReturn ret = GST_FLOW_OK; GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(parent); GstPMAudioVisualizerClass *klass; - GstClockTime ts, frame_duration; + GstClockTime ts; guint avail, sbpf; // databuf is a buffer holding one video frame worth of audio data used as // temp buffer for copying from the adapter only @@ -793,24 +793,23 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, inbuf = scope->priv->inbuf; - /* original code FIXME: the timestamp in the adapter would be different - this - * should be fixed now by deriving timestamps from the number of samples - * consumed. */ + // prepare buffer gst_buffer_copy_into(inbuf, buffer, GST_BUFFER_COPY_METADATA, 0, -1); /* this is what we have */ avail = gst_adapter_available(scope->priv->adapter); - // GST_LOG_OBJECT(scope, "avail: %u, bpf: %u", avail, sbpf); + while (avail >= sbpf) { gboolean fps_changed_since_last_frame = scope->priv->fps_changed; scope->priv->fps_changed = FALSE; // make sure frame duration does not change while processing one frame - frame_duration = scope->req_frame_duration; + const GstClockTime frame_duration = scope->req_frame_duration; - /* calculate timestamp based on audio input samples already processed to - * avoid clock drift */ + // derive timestamps from the number of samples consumed, + // calculate timestamp based on audio input samples already processed to + // avoid clock drift ts = scope->priv->pts_offset + gst_util_uint64_scale_int(scope->priv->samples_consumed, GST_SECOND, GST_AUDIO_INFO_RATE(&scope->ainfo)); @@ -859,8 +858,9 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, } } - /* map pts ts via segment for general use */ - ts = gst_segment_to_stream_time(&scope->priv->segment, GST_FORMAT_TIME, ts); + // map pts ts via segment to running time + ts = + gst_segment_to_running_time(&scope->priv->segment, GST_FORMAT_TIME, ts); ++scope->priv->processed; @@ -901,8 +901,8 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, gst_adapter_flush(scope->priv->adapter, sbpf); } else if (avail >= sbpf) { // was just enough audio data for one frame - /* just flush a bit and stop */ // rendering. seems like a bug in the original code + /* just flush a bit and stop */ // gst_adapter_flush(scope->priv->adapter, (avail - sbpf)); // instead just flush one video frame worth of audio data from the buffer diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index c3a55c0..d557db1 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -29,7 +29,8 @@ enum { PROP_PRESET_LOCKED, PROP_SHUFFLE_PRESETS, PROP_ENABLE_PLAYLIST, - PROP_MIN_FPS + PROP_MIN_FPS, + PROP_IS_LIVE }; /** @@ -51,6 +52,9 @@ enum { #define DEFAULT_ENABLE_PLAYLIST TRUE #define DEFAULT_SHUFFLE_PRESETS TRUE // depends on ENABLE_PLAYLIST #define DEFAULT_MIN_FPS "1/1" +#define DEFAULT_MIN_FPS_N 1 +#define DEFAULT_MIN_FPS_D 1 +#define DEFAULT_IS_LIVE "auto" void gst_projectm_base_init_once() { GST_DEBUG_CATEGORY_INIT(gst_projectm_base_debug, "projectm_base", 0, @@ -150,31 +154,33 @@ projectm_init(GObject *plugin, GstBaseProjectMSettings *settings, GST_DEBUG_OBJECT(plugin, "Playlist disabled"); } // Log properties - GST_INFO_OBJECT( - plugin, - "Using Properties: " - "preset=%s, " - "texture-dir=%s, " - "beat-sensitivity=%f, " - "hard-cut-duration=%f, " - "hard-cut-enabled=%d, " - "hard-cut-sensitivity=%f, " - "soft-cut-duration=%f, " - "preset-duration=%f, " - "mesh-size=(%lu, %lu)" - "aspect-correction=%d, " - "easter-egg=%f, " - "preset-locked=%d, " - "enable-playlist=%d, " - "shuffle-presets=%d, " - "min-fps=%d/%d", - settings->preset_path, settings->texture_dir_path, - settings->beat_sensitivity, settings->hard_cut_duration, - settings->hard_cut_enabled, settings->hard_cut_sensitivity, - settings->soft_cut_duration, settings->preset_duration, - settings->mesh_width, settings->mesh_height, settings->aspect_correction, - settings->easter_egg, settings->preset_locked, settings->enable_playlist, - settings->shuffle_presets, settings->min_fps_n, settings->min_fps_d); + GST_INFO_OBJECT(plugin, + "Using Properties: " + "preset=%s, " + "texture-dir=%s, " + "beat-sensitivity=%f, " + "hard-cut-duration=%f, " + "hard-cut-enabled=%d, " + "hard-cut-sensitivity=%f, " + "soft-cut-duration=%f, " + "preset-duration=%f, " + "mesh-size=(%lu, %lu)" + "aspect-correction=%d, " + "easter-egg=%f, " + "preset-locked=%d, " + "enable-playlist=%d, " + "shuffle-presets=%d, " + "min-fps=%d/%d, " + "is-live=%s", + settings->preset_path, settings->texture_dir_path, + settings->beat_sensitivity, settings->hard_cut_duration, + settings->hard_cut_enabled, settings->hard_cut_sensitivity, + settings->soft_cut_duration, settings->preset_duration, + settings->mesh_width, settings->mesh_height, + settings->aspect_correction, settings->easter_egg, + settings->preset_locked, settings->enable_playlist, + settings->shuffle_presets, settings->min_fps_n, + settings->min_fps_d, settings->is_live); // Load preset file if path is provided if (settings->preset_path != NULL) { @@ -314,6 +320,10 @@ void gst_projectm_base_set_property(GObject *object, g_object_set(G_OBJECT(glav), "min-fps-n", num, "min-fps-d", denom, NULL); } break; + case PROP_IS_LIVE: + settings->is_live = g_strdup(g_value_get_string(value)); + g_object_set(G_OBJECT(glav), "is-live", settings->is_live, NULL); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -386,6 +396,9 @@ void gst_projectm_base_get_property(GObject *object, g_object_set(G_OBJECT(glav), "min-fps-n", settings->min_fps_n, "min-fps-d", settings->min_fps_d, NULL); break; + case PROP_IS_LIVE: + g_value_set_string(value, settings->is_live); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); break; @@ -406,6 +419,9 @@ void gst_projectm_base_init(GstBaseProjectMSettings *settings, settings->preset_duration = DEFAULT_PRESET_DURATION; settings->enable_playlist = DEFAULT_ENABLE_PLAYLIST; settings->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; + settings->min_fps_d = DEFAULT_MIN_FPS_D; + settings->min_fps_n = DEFAULT_MIN_FPS_N; + settings->is_live = DEFAULT_IS_LIVE; const gchar *meshSizeStr = DEFAULT_MESH_SIZE; gint width, height; @@ -439,6 +455,7 @@ void gst_projectm_base_finalize(GstBaseProjectMSettings *settings, GstBaseProjectMPrivate *priv) { g_free(settings->preset_path); g_free(settings->texture_dir_path); + g_free(settings->is_live); g_mutex_clear(&priv->projectm_lock); } @@ -686,4 +703,15 @@ void gst_projectm_base_install_properties(GObjectClass *gobject_class) { "can't keep up with pipeline fps. Applies to real-time pipelines " "only.", DEFAULT_MIN_FPS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property( + gobject_class, PROP_IS_LIVE, + g_param_spec_string( + "is-live", "is live", + "Specifies if the plugin renders in real-time or as fast as possible " + "(offline). This setting is auto-detected and does not need to be " + "specified, but can be specified for cases where auto-detection is " + "not appropriate. Possible values are \"auto\", \"true\", \"false\". " + "Default is \"auto\".", + DEFAULT_IS_LIVE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h index 439d408..e7107e0 100644 --- a/src/gstprojectmbase.h +++ b/src/gstprojectmbase.h @@ -20,6 +20,7 @@ struct _GstBaseProjectMSettings { gchar *preset_path; gchar *texture_dir_path; + gchar *is_live; gfloat beat_sensitivity; gdouble hard_cut_duration; diff --git a/src/renderbuffer.c b/src/renderbuffer.c index ea2fba0..e3a7da8 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -168,7 +168,8 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, const RBAdjustFpsFunc adjust_fps_func, const GstClockTime max_frame_duration, const GstClockTime caps_frame_duration, - const gboolean is_qos_enabled) { + const gboolean is_qos_enabled, + const gboolean is_realtime) { GST_DEBUG_CATEGORY_INIT(renderbuffer_debug, "renderbuffer", 0, "projectM visualizer plugin render buffer"); @@ -182,6 +183,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, g_atomic_int_set(&state->running, FALSE); state->qos_enabled = is_qos_enabled; + state->is_realtime = is_realtime; state->caps_frame_duration = caps_frame_duration; state->max_frame_duration = max_frame_duration; @@ -193,6 +195,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, g_mutex_init(&state->slot_lock); g_cond_init(&state->slot_available_cond); g_cond_init(&state->render_queued_cond); + state->buffer_push_queue = g_async_queue_new(); state->buffer_cleanup_queue = g_async_queue_new(); for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { @@ -208,13 +211,14 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, } void rb_dispose_render_buffer(RBRenderBuffer *state) { + g_async_queue_unref(state->buffer_push_queue); g_async_queue_unref(state->buffer_cleanup_queue); g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); g_mutex_clear(&state->slot_lock); } -RBQueueResult rb_queue_render_job(RBQueueArgs *args) { +RBQueueResult rb_queue_render_task(RBQueueArgs *args) { RBRenderBuffer *state = args->render_buffer; const gboolean wait_is_limited = args->max_wait != GST_CLOCK_TIME_NONE; @@ -309,12 +313,12 @@ RBQueueResult rb_queue_render_job(RBQueueArgs *args) { return result; } -void rb_queue_render_job_log(RBQueueArgs *args) { +void rb_queue_render_task_log(RBQueueArgs *args) { RBRenderBuffer *state = args->render_buffer; const GstClockTime start_ts = gst_util_get_timestamp(); - const RBQueueResult result = rb_queue_render_job(args); + const RBQueueResult result = rb_queue_render_task(args); switch (result) { case RB_EVICTED: { @@ -393,13 +397,13 @@ static void gl_buffer_cleanup(GstGLContext *context, gpointer buf) { } /** - * Used to dispose of gl buffers only. - * Consume buffers to clean-up and dispatches release through gl thread. + * Used to dispose of dropped gl buffers only. + * Consume buffers to clean-up and dispatch release through gl thread. * - * @param user_data The GstBuffer pointer to release. - * @return + * @param user_data Render buffer to use. + * @return NULL */ -static gpointer cleanup_thread_func(gpointer user_data) { +static gpointer rb_cleanup_thread_func(gpointer user_data) { const RBRenderBuffer *state = (RBRenderBuffer *)user_data; // consume gl buffers to dispatch to gl thread for cleanup @@ -446,6 +450,46 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { return render_time; } +/** + * Pushes gl buffers for real-time rendering only. + * Consume buffers to push and wait until it's time to push. + * + * @param user_data Render buffer to use. + * @return NULL + */ +static gpointer rb_push_thread_func(gpointer user_data) { + + const RBRenderBuffer *state = (RBRenderBuffer *)user_data; + + while (g_atomic_int_get(&state->running)) { + + // consume gl buffer to push + gpointer item = g_async_queue_pop(state->buffer_push_queue); + + if (!item || item == RB_Q_SHUTDOWN_SIGNAL) + continue; + + GstBuffer *outbuf = GST_BUFFER(item); + + // buffers are in PTS order, wait until it's time to push + GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + if (clock) { + GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + GstClockTime abs_time = GST_BUFFER_PTS(outbuf) + base_time; + gst_clock_id_wait(gst_clock_new_single_shot_id(clock, abs_time), NULL); + gst_object_unref(clock); + } + + // push buffer downstream + const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); + if (ret != GST_FLOW_OK) { + GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); + } + } + return NULL; +} + /** * Send a video buffer to the source pad downstream. * Buffer is checked and timestamps are populated before sending. @@ -470,10 +514,20 @@ GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, GST_BUFFER_DTS(outbuf) = pts; GST_BUFFER_DURATION(outbuf) = frame_duration; - // push buffer downstream - const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); - if (ret != GST_FLOW_OK) { - GST_WARNING("Failed to push buffer to pad"); + GstFlowReturn ret; + if (state->is_realtime) { + // for real-time, we need to wait until it's time to push the buffer + // dispatch the wait and push to another thread to keep rendering as fast + // as we can (and get audio buffers) + g_async_queue_push(state->buffer_push_queue, outbuf); + ret = GST_FLOW_OK; + } else { + // push buffer downstream directly for offline rendering, avoid scheduling + // overhead + ret = gst_pad_push(state->src_pad, outbuf); + if (ret != GST_FLOW_OK) { + GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); + } } return ret; } @@ -642,30 +696,44 @@ static gpointer rb_render_thread_func(gpointer user_data) { return NULL; } -void rb_start_render_thread(RBRenderBuffer *state, GstGLContext *gl_context, - GstPad *src_pad) { +void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, + GstPad *src_pad) { state->gl_context = gl_context; state->src_pad = src_pad; g_atomic_int_set(&state->running, TRUE); - state->render_thread = - g_thread_new("rb-render-thread", rb_render_thread_func, state); - state->cleanup_thread = - g_thread_new("rb-cleanup-thread", cleanup_thread_func, state); + + // threads are not needed for offline rendering + if (state->is_realtime) { + state->render_thread = + g_thread_new("rb-render-thread", rb_render_thread_func, state); + state->push_thread = + g_thread_new("rb-push-thread", rb_push_thread_func, state); + state->cleanup_thread = + g_thread_new("rb-cleanup-thread", rb_cleanup_thread_func, state); + } GST_INFO_OBJECT(state->plugin, "Started render buffer"); } -void rb_stop_render_thread(RBRenderBuffer *state) { +void rb_stop(RBRenderBuffer *state) { - g_mutex_lock(&state->slot_lock); g_atomic_int_set(&state->running, FALSE); - g_cond_broadcast(&state->render_queued_cond); - g_mutex_unlock(&state->slot_lock); + // threads are not needed for offline rendering + if (state->is_realtime) { + g_mutex_lock(&state->slot_lock); + g_cond_broadcast(&state->render_queued_cond); + g_mutex_unlock(&state->slot_lock); + + g_thread_join(state->render_thread); + + g_async_queue_push(state->buffer_push_queue, RB_Q_SHUTDOWN_SIGNAL); + g_thread_join(state->push_thread); + + g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); + g_thread_join(state->cleanup_thread); + } - g_thread_join(state->render_thread); - g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); - g_thread_join(state->cleanup_thread); state->gl_context = NULL; state->src_pad = NULL; GST_INFO_OBJECT(state->plugin, "Stopped render buffer"); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 1ba533a..54cdb85 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -205,6 +205,18 @@ typedef struct { */ GThread *cleanup_thread; + /** + * Thread for pushing gl buffers downstream. + * Used for real-time, pushing needs to be scheduled to be synchronized with + * the pipeline clock. + */ + GThread *push_thread; + + /** + * Queue to schedule gl buffers for pushing. + */ + GAsyncQueue *buffer_push_queue; + /** * Queue to dispose of dropped gl buffers. */ @@ -241,10 +253,15 @@ typedef struct { GCond slot_available_cond; /** - * Is current pipeline using a real-time clock. + * Switch for real-time (render loop) QoS. */ gboolean qos_enabled; + /** + * Is current pipeline using a real-time clock. + */ + gboolean is_realtime; + /** * Pipeline negotiated caps fps as frame duration. */ @@ -336,7 +353,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, RBAdjustFpsFunc adjust_fps_func, GstClockTime max_frame_duration, GstClockTime caps_frame_duration, - gboolean is_qos_enabled); + gboolean is_qos_enabled, gboolean is_realtime); /** * Release resources for the given render buffer. @@ -354,7 +371,7 @@ void rb_dispose_render_buffer(RBRenderBuffer *state); * does not take ownership of the given pointer. The given audio buffer is * copied. */ -RBQueueResult rb_queue_render_job(RBQueueArgs *args); +RBQueueResult rb_queue_render_task(RBQueueArgs *args); /** * Queue an audio buffer for rendering. The queuing is guaranteed to return @@ -368,7 +385,7 @@ RBQueueResult rb_queue_render_job(RBQueueArgs *args); * does not take ownership of the given pointer. The given audio buffer is * copied. */ -void rb_queue_render_job_log(RBQueueArgs *args); +void rb_queue_render_task_log(RBQueueArgs *args); /** * Render one frame synchronously. Using synchronous rendering is exclusive, @@ -403,15 +420,14 @@ static gboolean rb_is_render_too_late(GstElement *element, GstClockTime latency, * @param gl_context GL context to use for rendering. * @param src_pad Source pad to push video buffers to. */ -void rb_start_render_thread(RBRenderBuffer *state, GstGLContext *gl_context, - GstPad *src_pad); +void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, GstPad *src_pad); /** * Stop render loop. Active threads will be joined before returning. * * @param state Render buffer to use. */ -void rb_stop_render_thread(RBRenderBuffer *state); +void rb_stop(RBRenderBuffer *state); /** * Update caps as they get negotiated by the pipeline. Thread safe. @@ -422,15 +438,6 @@ void rb_stop_render_thread(RBRenderBuffer *state); void rb_set_caps_frame_duration(RBRenderBuffer *state, GstClockTime caps_frame_duration); -/** - * Controls if real-time QoS is enabled. Thread safe. - * Should be enabled if the pipeline is using a real-time clock. - * - * @param state Render buffer to update. - * @param is_qos_enabled TRUE real-time QoS is enabled. - */ -void rb_set_qos_enabled(RBRenderBuffer *state, gboolean is_qos_enabled); - G_END_DECLS #endif // __RENDERBUFFER_H__ From 81659d324da189e61f67bc30fe4d2e31bdaf2e8a Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Fri, 10 Oct 2025 02:24:52 -0500 Subject: [PATCH 11/32] fix real-time detection --- src/gstglbaseaudiovisualizer.c | 95 ++++++++++++++++++---------------- src/gstprojectmbase.c | 9 ++-- 2 files changed, 57 insertions(+), 47 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 74959e0..3ced311 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -403,6 +403,34 @@ static void gst_gl_base_audio_visualizer_set_context(GstElement *element, GST_ELEMENT_CLASS(parent_class)->set_context(element, context); } +/** + * Find the pipeline and determine if it is live. + * + * @param element Plugin element. + * + * @return TRUE if the pipeline is live. + */ +static gboolean is_pipeline_live(GstElement *element) { + GstPipeline *pipeline = NULL; + gboolean is_live = FALSE; + + GstObject *parent = GST_OBJECT(element); + while (parent && !GST_IS_PIPELINE(parent)) { + GstObject *next = gst_object_get_parent(parent); + if (parent != GST_OBJECT(element)) + gst_object_unref(parent); + parent = next; + } + + if (parent && GST_IS_PIPELINE(parent)) { + pipeline = GST_PIPELINE(parent); + is_live = gst_pipeline_is_live(pipeline); + gst_object_unref(parent); + } + + return is_live; +} + static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav) { return TRUE; @@ -437,6 +465,16 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, gst_util_uint64_scale_int(GST_SECOND, GST_VIDEO_INFO_FPS_D(&pmav->vinfo), GST_VIDEO_INFO_FPS_N(&pmav->vinfo)); + // determine if we're using a real-time pipeline + if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE) { + glav->priv->is_realtime = FALSE; + } else if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_REALTIME) { + glav->priv->is_realtime = TRUE; + } else { + // auto detect + glav->priv->is_realtime = is_pipeline_live(GST_ELEMENT(data)); + } + // render loop QoS is disabled for offline rendering rb_init_render_buffer(&glav->priv->render_buffer, GST_OBJECT(glav), gst_gl_base_audio_visualizer_fill_gl, @@ -870,40 +908,23 @@ static gboolean gst_gl_base_audio_visualizer_parent_decide_allocation( return TRUE; } -/** - * Find pipeline clock and determine if it is a real-time clock. - * - * @param element Plugin element. - * - * @return TRUE if the pipeline clock is a real-time (system) clock. - */ -static gboolean is_pipeline_realtime(GstElement *element) { - GstClock *clock = NULL; - gboolean is_realtime = FALSE; - - // first check element's own clock, then start climbing the hierarchy - clock = gst_element_get_clock(element); - if (!clock) { - // traverse parents to find the pipeline and ask it for its clock - GstObject *parent = gst_object_get_parent(GST_OBJECT(element)); - while (parent && !GST_IS_PIPELINE(parent)) { - GstObject *next_parent = gst_object_get_parent(parent); - gst_object_unref(parent); - parent = next_parent; - } - if (parent && GST_IS_PIPELINE(parent)) { - GstPipeline *pipeline = GST_PIPELINE(parent); - clock = gst_pipeline_get_clock(pipeline); +static GstPipeline *get_pipeline(GstElement *element) { + GstObject *parent = GST_OBJECT(element); + + while (parent) { + if (GST_IS_PIPELINE(parent)) + return GST_PIPELINE(parent); + + GstObject *next = gst_object_get_parent(parent); + + // we increase ref with get_parent, so unref previous level + if (parent != GST_OBJECT(element)) gst_object_unref(parent); - } - } - if (clock) { - is_realtime = GST_IS_SYSTEM_CLOCK(clock); - gst_object_unref(clock); + parent = next; } - return is_realtime; + return NULL; // no pipeline found } static GstStateChangeReturn @@ -925,20 +946,6 @@ gst_gl_base_audio_visualizer_change_state(GstElement *element, g_rec_mutex_unlock(&glav->priv->context_lock); break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - g_rec_mutex_lock(&glav->priv->context_lock); - // determine if we're using a real-time pipeline - if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE) { - glav->priv->is_realtime = FALSE; - } else if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_REALTIME) { - glav->priv->is_realtime = TRUE; - } else { - // auto detect - glav->priv->is_realtime = is_pipeline_realtime(element); - } - g_rec_mutex_unlock(&glav->priv->context_lock); - break; - default: break; } diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index d557db1..73e706e 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -255,9 +255,11 @@ void gst_projectm_base_set_property(GObject *object, switch (property_id) { case PROP_PRESET_PATH: + g_free(settings->preset_path); settings->preset_path = g_strdup(g_value_get_string(value)); break; case PROP_TEXTURE_DIR_PATH: + g_free(settings->texture_dir_path); settings->texture_dir_path = g_strdup(g_value_get_string(value)); break; case PROP_BEAT_SENSITIVITY: @@ -321,6 +323,7 @@ void gst_projectm_base_set_property(GObject *object, } break; case PROP_IS_LIVE: + g_free(settings->is_live); settings->is_live = g_strdup(g_value_get_string(value)); g_object_set(G_OBJECT(glav), "is-live", settings->is_live, NULL); break; @@ -421,7 +424,7 @@ void gst_projectm_base_init(GstBaseProjectMSettings *settings, settings->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; settings->min_fps_d = DEFAULT_MIN_FPS_D; settings->min_fps_n = DEFAULT_MIN_FPS_N; - settings->is_live = DEFAULT_IS_LIVE; + settings->is_live = strdup(DEFAULT_IS_LIVE); const gchar *meshSizeStr = DEFAULT_MESH_SIZE; gint width, height; @@ -709,8 +712,8 @@ void gst_projectm_base_install_properties(GObjectClass *gobject_class) { g_param_spec_string( "is-live", "is live", "Specifies if the plugin renders in real-time or as fast as possible " - "(offline). This setting is auto-detected and does not need to be " - "specified, but can be specified for cases where auto-detection is " + "(offline). This setting is auto-detected for live pipelines, " + "but can also be specified if auto-detection is " "not appropriate. Possible values are \"auto\", \"true\", \"false\". " "Default is \"auto\".", DEFAULT_IS_LIVE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); From 3357bde2b4026a7de33e772fd9391d466551ffa2 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 11 Oct 2025 00:14:26 -0500 Subject: [PATCH 12/32] fix unbounded buffer queue, throttle real-time rendering --- src/renderbuffer.c | 156 +++++++++++++++++++++++++++++++++++++-------- src/renderbuffer.h | 33 +++++++++- 2 files changed, 162 insertions(+), 27 deletions(-) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index e3a7da8..aa8c29b 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -25,7 +25,7 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; */ #ifndef RB_EMA_ALPHA_N #define RB_EMA_ALPHA_N 1 -#define RB_EMA_ALPHA_D 4 +#define RB_EMA_ALPHA_D 5 #endif /** @@ -60,8 +60,8 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; * allow render time as low as 0.9x */ #ifndef RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N -#define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N 9 -#define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D 10 +#define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N 95 +#define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D 100 #endif /** @@ -157,12 +157,6 @@ static void rb_queue_gl_buffer_cleanup(const RBRenderBuffer *state, g_async_queue_push(state->buffer_cleanup_queue, out); } -void rb_set_qos_enabled(RBRenderBuffer *state, const gboolean is_qos_enabled) { - g_mutex_lock(&state->slot_lock); - state->qos_enabled = is_qos_enabled; - g_mutex_unlock(&state->slot_lock); -} - void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, const GstGLContextThreadFunc gl_fill_func, const RBAdjustFpsFunc adjust_fps_func, @@ -191,12 +185,18 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, state->last_render_index = -1; state->frame_counter = 0; state->smoothed_render_time = 0; + state->buffer_push_queue_read_idx = -1; + state->buffer_push_queue_write_idx = -1; g_mutex_init(&state->slot_lock); + g_mutex_init(&state->buffer_push_queue_mutex); g_cond_init(&state->slot_available_cond); g_cond_init(&state->render_queued_cond); - state->buffer_push_queue = g_async_queue_new(); + g_cond_init(&state->buffer_push_queue_cond); state->buffer_cleanup_queue = g_async_queue_new(); + for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + state->buffer_push_queue[i] = NULL; + } for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { state->slots[i].state = RB_EMPTY; @@ -211,11 +211,18 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, } void rb_dispose_render_buffer(RBRenderBuffer *state) { - g_async_queue_unref(state->buffer_push_queue); + for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + if (state->buffer_push_queue[i] != NULL) { + rb_queue_gl_buffer_cleanup(state, state->buffer_push_queue[i]); + state->buffer_push_queue[i] = NULL; + } + } g_async_queue_unref(state->buffer_cleanup_queue); g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); + g_cond_clear(&state->buffer_push_queue_cond); g_mutex_clear(&state->slot_lock); + g_mutex_clear(&state->buffer_push_queue_mutex); } RBQueueResult rb_queue_render_task(RBQueueArgs *args) { @@ -450,6 +457,35 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { return render_time; } +/** + * Schedule a rendered gl buffer for pushing downstream. + * The buffer will not be pushed until it's PTS time is reached. + * Subsequent queued buffers will override existing buffers if they are not due + * for pushing in time. + * + * @param state The render buffer to use. + * @param buffer The gl buffer to push. Takes ownership of the buffer. + */ +static void rb_schedule_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { + g_mutex_lock(&state->buffer_push_queue_mutex); + + // just writer to the next position in the ring, no matter what + state->buffer_push_queue_write_idx = + (state->buffer_push_queue_write_idx + 1) % PUSH_QUEUE_MAX_SIZE; + + // dispose buffer, if any + if (state->buffer_push_queue[state->buffer_push_queue_write_idx] != NULL) { + rb_queue_gl_buffer_cleanup( + state, state->buffer_push_queue[state->buffer_push_queue_write_idx]); + } + + // take the spot and signal that queue has changed + state->buffer_push_queue[state->buffer_push_queue_write_idx] = buffer; + g_cond_signal(&state->buffer_push_queue_cond); + + g_mutex_unlock(&state->buffer_push_queue_mutex); +} + /** * Pushes gl buffers for real-time rendering only. * Consume buffers to push and wait until it's time to push. @@ -459,34 +495,89 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { */ static gpointer rb_push_thread_func(gpointer user_data) { - const RBRenderBuffer *state = (RBRenderBuffer *)user_data; + RBRenderBuffer *state = (RBRenderBuffer *)user_data; + + // buffers are in PTS order, wait until it's time to push + + GstClockTimeDiff used_wait; + GstClockTimeDiff frame_wait; + GstClockTime wait_start; + + g_mutex_lock(&state->buffer_push_queue_mutex); while (g_atomic_int_get(&state->running)) { - // consume gl buffer to push - gpointer item = g_async_queue_pop(state->buffer_push_queue); + state->buffer_push_queue_read_idx = + (state->buffer_push_queue_read_idx + 1) % PUSH_QUEUE_MAX_SIZE; - if (!item || item == RB_Q_SHUTDOWN_SIGNAL) + // consume gl buffer to push + if (state->buffer_push_queue[state->buffer_push_queue_read_idx] == NULL) { + // no buffer to push, wait for one + state->buffer_push_queue_read_idx = state->buffer_push_queue_write_idx; + g_cond_wait(&state->buffer_push_queue_cond, + &state->buffer_push_queue_mutex); continue; + } - GstBuffer *outbuf = GST_BUFFER(item); + // the buffer to push (for now, we may still lose it) + GstBuffer *outbuf = + state->buffer_push_queue[state->buffer_push_queue_read_idx]; - // buffers are in PTS order, wait until it's time to push + // determine when it's time to push GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - GstClockTime abs_time = GST_BUFFER_PTS(outbuf) + base_time; - gst_clock_id_wait(gst_clock_new_single_shot_id(clock, abs_time), NULL); - gst_object_unref(clock); + GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + + const GstClockTime pts = GST_BUFFER_PTS(outbuf); + const GstClockTime abs_time = pts + base_time; + + wait_start = gst_clock_get_time(clock); + frame_wait = GST_CLOCK_DIFF(wait_start, abs_time); + used_wait = 0; + gboolean abort = FALSE; + + // interruptable wait, continues when ring buffer read pointer is overtaken + // by write pointer while waiting + while (used_wait < frame_wait) { + + abort = !g_atomic_int_get(&state->running); + if (abort) { + break; + } + + // wait until it's time to push or another buffer is queued + g_cond_wait_until(&state->buffer_push_queue_cond, + &state->buffer_push_queue_mutex, + (GstClockTimeDiff)wait_start + frame_wait - used_wait); + used_wait = (GstClockTimeDiff)gst_clock_get_time(clock) - wait_start; + + // were we overtaken ? + if (state->buffer_push_queue[state->buffer_push_queue_read_idx] != + outbuf) { + abort = TRUE; + break; + } } + gst_object_unref(clock); + + if (abort) { + continue; + } + + // now we own the buffer to push + state->buffer_push_queue[state->buffer_push_queue_read_idx] = NULL; + + g_mutex_unlock(&state->buffer_push_queue_mutex); // push buffer downstream const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } + g_mutex_lock(&state->buffer_push_queue_mutex); } + g_mutex_unlock(&state->buffer_push_queue_mutex); + return NULL; } @@ -519,7 +610,7 @@ GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, // for real-time, we need to wait until it's time to push the buffer // dispatch the wait and push to another thread to keep rendering as fast // as we can (and get audio buffers) - g_async_queue_push(state->buffer_push_queue, outbuf); + rb_schedule_push_buffer(state, outbuf); ret = GST_FLOW_OK; } else { // push buffer downstream directly for offline rendering, avoid scheduling @@ -658,6 +749,21 @@ static gpointer rb_render_thread_func(gpointer user_data) { g_mutex_unlock(&state->slot_lock); + // throttle rendering + GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + if (clock) { + GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + GstClockTime now = gst_clock_get_time(clock) - base_time; + + // wait if we're too far ahead + if (now < (GstClockTimeDiff)slot->pts - slot->frame_duration) { + gst_clock_id_wait( + gst_clock_new_single_shot_id(clock, base_time + slot->pts), NULL); + } + } + gst_object_unref(clock); + // perform gl rendering GstClockTime render_time = rb_render_slot(state, slot); @@ -727,7 +833,7 @@ void rb_stop(RBRenderBuffer *state) { g_thread_join(state->render_thread); - g_async_queue_push(state->buffer_push_queue, RB_Q_SHUTDOWN_SIGNAL); + rb_schedule_push_buffer(state, NULL); g_thread_join(state->push_thread); g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 54cdb85..9f7bf69 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -65,6 +65,15 @@ G_BEGIN_DECLS #define NUM_RENDER_SLOTS 2 #endif +/** + * Max frames waiting in a scheduled state to be pushed. + * Should be short, will drop frames if newer frame are queued, last frame wins. + * Render loop if real-time capped, there should not be many buffers waiting. + */ +#ifndef PUSH_QUEUE_MAX_SIZE +#define PUSH_QUEUE_MAX_SIZE 2 +#endif + /** * Callback function pointer type for triggering a dynamic fps change. */ @@ -213,9 +222,29 @@ typedef struct { GThread *push_thread; /** - * Queue to schedule gl buffers for pushing. + * Ring buffer to schedule gl buffers for pushing. + */ + GstBuffer *buffer_push_queue[PUSH_QUEUE_MAX_SIZE]; + + /** + * Push ring buffer write position. + */ + gint buffer_push_queue_write_idx; + + /** + * Push ring buffer read position. + */ + gint buffer_push_queue_read_idx; + + /** + * Mutex for push ring buffer. + */ + GMutex buffer_push_queue_mutex; + + /** + * Condition as interruptable scheduling clock for push ring buffer. */ - GAsyncQueue *buffer_push_queue; + GCond buffer_push_queue_cond; /** * Queue to dispose of dropped gl buffers. From f210200164b4e1ff713101094976e56b3f6f58d6 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 11 Oct 2025 01:35:53 -0500 Subject: [PATCH 13/32] fix busy wait, fix missing buffer cleanups --- src/renderbuffer.c | 61 +++++++++++++++++++++++++++++++--------------- src/renderbuffer.h | 13 ++++++++-- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index aa8c29b..d95c44c 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -211,12 +211,6 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, } void rb_dispose_render_buffer(RBRenderBuffer *state) { - for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { - if (state->buffer_push_queue[i] != NULL) { - rb_queue_gl_buffer_cleanup(state, state->buffer_push_queue[i]); - state->buffer_push_queue[i] = NULL; - } - } g_async_queue_unref(state->buffer_cleanup_queue); g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); @@ -469,7 +463,7 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { static void rb_schedule_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { g_mutex_lock(&state->buffer_push_queue_mutex); - // just writer to the next position in the ring, no matter what + // just write to the next position in the ring, no matter what state->buffer_push_queue_write_idx = (state->buffer_push_queue_write_idx + 1) % PUSH_QUEUE_MAX_SIZE; @@ -546,9 +540,10 @@ static gpointer rb_push_thread_func(gpointer user_data) { } // wait until it's time to push or another buffer is queued + // (microseconds!) g_cond_wait_until(&state->buffer_push_queue_cond, &state->buffer_push_queue_mutex, - (GstClockTimeDiff)wait_start + frame_wait - used_wait); + g_get_real_time() + (frame_wait - used_wait) / 1000); used_wait = (GstClockTimeDiff)gst_clock_get_time(clock) - wait_start; // were we overtaken ? @@ -576,6 +571,15 @@ static gpointer rb_push_thread_func(gpointer user_data) { } g_mutex_lock(&state->buffer_push_queue_mutex); } + + // release buffers that are still queued before cleanup thread shuts down + for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + if (state->buffer_push_queue[i] != NULL) { + rb_queue_gl_buffer_cleanup(state, state->buffer_push_queue[i]); + state->buffer_push_queue[i] = NULL; + } + } + g_mutex_unlock(&state->buffer_push_queue_mutex); return NULL; @@ -584,6 +588,7 @@ static gpointer rb_push_thread_func(gpointer user_data) { /** * Send a video buffer to the source pad downstream. * Buffer is checked and timestamps are populated before sending. + * Push is blocking for offline rendering, and queued for real-time rendering. * * @param state Render buffer to use. * @param outbuf Video buffer to send downstream (takes ownership). @@ -750,19 +755,21 @@ static gpointer rb_render_thread_func(gpointer user_data) { g_mutex_unlock(&state->slot_lock); // throttle rendering - GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - GstClockTime now = gst_clock_get_time(clock) - base_time; - - // wait if we're too far ahead - if (now < (GstClockTimeDiff)slot->pts - slot->frame_duration) { - gst_clock_id_wait( - gst_clock_new_single_shot_id(clock, base_time + slot->pts), NULL); + if (state->is_realtime) { + GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + if (clock) { + GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + GstClockTime now = gst_clock_get_time(clock) - base_time; + + // wait if we're too far ahead + if (now < (GstClockTimeDiff)slot->pts - slot->frame_duration) { + gst_clock_id_wait( + gst_clock_new_single_shot_id(clock, base_time + slot->pts), NULL); + } } + gst_object_unref(clock); } - gst_object_unref(clock); // perform gl rendering GstClockTime render_time = rb_render_slot(state, slot); @@ -799,6 +806,22 @@ static gpointer rb_render_thread_func(gpointer user_data) { } } + g_mutex_lock(&state->slot_lock); + for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { + if (state->slots[i].state == RB_READY) { + if (state->slots[i].in_audio) { + gst_buffer_unref(state->slots[i].in_audio); + state->slots[i].in_audio = NULL; + } + if (state->slots[i].out_buf) { + rb_queue_gl_buffer_cleanup(state, state->slots[i].out_buf); + state->slots[i].out_buf = NULL; + } + state->slots[i].state = RB_EMPTY; + } + } + g_mutex_unlock(&state->slot_lock); + return NULL; } diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 9f7bf69..61f817c 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -38,6 +38,14 @@ * - If the render duration exceeds the fps *most of the time*, an Exponential * Moving Average (EMA) based algorithm instructs the plugin to reduce fps. * EMA will also recover fps when render performance increases again. + * + * - Buffers that completed rendering are scheduled to be pushed to the source + * pad at PTS. A ring buffer queues buffers for pushing. The buffer acts like + * a blocking queue with scheduling wait. A separate worker thread consumes + * the ring buffer and waits for reaching the PTS of the current buffer. The + * wait is interruptable, in case the read pointer is overtaken by the write + * pointer, waiting for PTS is aborted and the next frame is scheduled + * immediately. */ #ifndef __RENDERBUFFER_H__ @@ -66,9 +74,10 @@ G_BEGIN_DECLS #endif /** - * Max frames waiting in a scheduled state to be pushed. + * Max number of frames waiting in a scheduled state to be pushed. * Should be short, will drop frames if newer frame are queued, last frame wins. - * Render loop if real-time capped, there should not be many buffers waiting. + * Render loop is real-time capped, there should not be a lot of buffers + * waiting. */ #ifndef PUSH_QUEUE_MAX_SIZE #define PUSH_QUEUE_MAX_SIZE 2 From c11b2f2fd27393785947c174da9f21e194575305 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 11 Oct 2025 02:41:00 -0500 Subject: [PATCH 14/32] increase push queue size to 3 --- src/renderbuffer.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 61f817c..3f8da1c 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -67,20 +67,20 @@ G_BEGIN_DECLS * * Valid values: * 1 - Wait for previous render to complete before scheduling. - * 2 - Render and schedule one item and at the same time. + * 2 - Render one item and schedule another at the same time. */ #ifndef NUM_RENDER_SLOTS #define NUM_RENDER_SLOTS 2 #endif /** - * Max number of frames waiting in a scheduled state to be pushed. - * Should be short, will drop frames if newer frame are queued, last frame wins. - * Render loop is real-time capped, there should not be a lot of buffers - * waiting. + * Max number of gl frame buffers waiting in a scheduled state to be pushed. + * Capacity should be low. Frames will be dropped if newer frame are queued, + * last frame wins. Render loop is capped for real-time, there should not be a + * lot of buffers waiting. */ #ifndef PUSH_QUEUE_MAX_SIZE -#define PUSH_QUEUE_MAX_SIZE 2 +#define PUSH_QUEUE_MAX_SIZE 3 #endif /** From c4e9aaec96ce948f73c42e9adc21229782098e72 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 11 Oct 2025 17:42:48 -0500 Subject: [PATCH 15/32] render buffer clean up, update docs --- README.md | 32 +-- src/gstglbaseaudiovisualizer.c | 15 +- src/gstprojectmbase.c | 2 +- src/renderbuffer.c | 378 +++++++++++++++++++++------------ src/renderbuffer.h | 109 ++++++---- 5 files changed, 340 insertions(+), 196 deletions(-) diff --git a/README.md b/README.md index a19e1c1..dd8154d 100644 --- a/README.md +++ b/README.md @@ -57,15 +57,15 @@ The documentation has been organized into distinct files, each dedicated to a sp - **[OSX](docs/OSX.md)** - **[Windows](docs/WINDOWS.md)** -Once the plugin has been installed, you can use it something like this to render in real-time to an OpenGL window: +Once the plugin has been installed, you can use it something like this to render to an OpenGL window: ```shell -gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false +gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=10 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false ``` To render from a live source in real-time to a gl window, an identity element can be used to provide a proper timestamp source for the pipeline. This example also includes a texture directory: ```shell -gst-launch souphttpsrc location=http://your-radio-stream is-live=true ! queue ! decodebin ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! identity single-segment=true sync=true ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 texture-dir=/usr/local/share/projectM/presets-milkdrop-texture-pack ! video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! glimagesink sync=false +gst-launch souphttpsrc location=http://your-radio-stream is-live=true ! queue ! decodebin ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! identity single-segment=true sync=true ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 is-live=true texture-dir=/usr/local/share/projectM/presets-milkdrop-texture-pack ! video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! glimagesink sync=false ``` Or to convert an audio file to video using offline rendering: @@ -77,7 +77,7 @@ filesrc location=input.mp3 ! decodebin name=dec \ t. ! queue ! audioconvert ! audioresample ! \ capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \ t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \ - projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ + projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \ identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \ gldownload \ x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ @@ -93,7 +93,7 @@ gst-launch-1.0 -e \ capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \ avenc_aac bitrate=320000 ! queue ! mux. \ t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! projectm \ - preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ + preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \ identity sync=false ! videoconvert ! videorate ! \ video/x-raw\(memory:GLMemory\),framerate=60/1,width=1920,height=1080 ! \ nvh264enc ! h264parse ! \ @@ -230,7 +230,7 @@ gst-launch-1.0 -e \ t. ! queue ! audioconvert ! audioresample ! \ capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \ t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \ - projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \ + projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \ identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \ gldownload \ x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \ @@ -249,15 +249,15 @@ gst-inspect projectm ## Technical Details -### 🖼️ OpenGL Rendering and Buffer Handling +### OpenGL Rendering and Buffer Handling - projectM output is rendered to OpenGL textures via **Frame Buffer Object (FBO)**. -- **Textures are pooled** and reused across frames to avoid excessive GPU memory allocation and de-allocation. +- **Textures are pooled** and reused across frames. - Each rendered texture becomes a GStreamer video buffer pushed downstream. **All video buffers stay in GPU memory**. --- -### ⏱️ Timing and Synchronization +### Timing and Synchronization The plugin synchronizes rendering to the GStreamer pipeline clock using **audio presentation timestamp (PTS) as the leading reference**. @@ -266,9 +266,11 @@ GStreamer's pipeline timing concept, and to enable faster-than-real-time renderi A **fixed number of audio samples is consumed per video frame**. **Example:** `735 samples per frame at 44.1 kHz = ~60 FPS.` - -Real-time pipelines only: Frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with +**Note:** Live pipelines are auto-detected by the plugin. For cases where auto-detection is not appropriate, +the `is-live` property can be configured. + +**Live pipelines only:** Frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with pipeline caps FPS. Video frame PTS offset is derived from the **first audio buffer PTS** or **segment event** plus accumulated samples to align with audio timing. @@ -279,10 +281,10 @@ Video frame PTS offset is derived from the **first audio buffer PTS** or **segme | Audio Timestamps | Audio Input | Always | Determine video timing and sync. | | Sample Rate / Pipeline FPS | Audio Input / Caps | Always | Defines how many audio samples are used per frame and target FPS. | | Segment Info | Segment Event | Always | Tracks running time and playback position. Used for PTS offsets. | -| QoS Feedback | QoS Event | Real-time | Skips outdated frames to reduce latency. | -| Render Frame Drop | Render Loop | Real-time | Drop frames that cannot be rendered in time. | -| Exponential Moving Average (EMA) | Render Loop | Real-time | Adjust plugin target FPS in case frame render time exceeds the real-time budget most of the time. | -| Latency Event | Render Loop | Real-time | Inform upstream of latency changes in case of adaptive FPS changes (EMA). | +| QoS Feedback | QoS Event | Live | Skips outdated frames to reduce latency. | +| Render Frame Drop | Render Loop | Live | Drop frames that cannot be rendered in time. | +| Exponential Moving Average (EMA) | Render Loop | Live | Adjust plugin target FPS in case frame render time exceeds the real-time budget most of the time. | +| Latency Event | Render Loop | Live | Inform upstream of latency changes in case of adaptive FPS changes (EMA). | --- diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 3ced311..900acab 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -81,7 +81,7 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); #define DEFAULT_MIN_FPS_N 1 #define DEFAULT_MIN_FPS_D 1 -#define DEFAULT_IS_LIVE "auto" +#define DEFAULT_PIPELINE_LIVE "auto" struct _GstGLBaseAudioVisualizerPrivate { GstGLContext *other_context; @@ -97,7 +97,7 @@ struct _GstGLBaseAudioVisualizerPrivate { }; /* Properties */ -enum { PROP_0, PROP_MIN_FPS_N, PROP_MIN_FPS_D, PROP_IS_LIVE }; +enum { PROP_0, PROP_MIN_FPS_N, PROP_MIN_FPS_D, PROP_PIPELINE_LIVE }; #define gst_gl_base_audio_visualizer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE_WITH_CODE( @@ -255,12 +255,12 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property( - gobject_class, PROP_IS_LIVE, - g_param_spec_string("is-live", "Is Live", + gobject_class, PROP_PIPELINE_LIVE, + g_param_spec_string("pipeline-live", "Pipeline Live", "Specifies if this element renders in real-time " "(true) or as fast as possible for offline rendering " "(false) or to auto-detect pipeline clock (auto)", - DEFAULT_IS_LIVE, + DEFAULT_PIPELINE_LIVE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } @@ -324,7 +324,7 @@ static void gst_gl_base_audio_visualizer_set_property(GObject *object, glav->min_fps_d = g_value_get_int(value); break; - case PROP_IS_LIVE: + case PROP_PIPELINE_LIVE: const char *str = g_value_get_string(value); if (strcasecmp("true", str) == 0) { glav->is_live = GST_GL_BASE_AUDIO_VISUALIZER_REALTIME; @@ -357,7 +357,7 @@ static void gst_gl_base_audio_visualizer_get_property(GObject *object, g_value_set_int(value, glav->min_fps_d); break; - case PROP_IS_LIVE: + case PROP_PIPELINE_LIVE: if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_REALTIME) { g_value_set_string(value, "true"); } else if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE) { @@ -504,6 +504,7 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, // stop gl rendering first rb_stop(&glav->priv->render_buffer); + rb_clear(&glav->priv->render_buffer); // clean up implementor if (glav->priv->gl_started) { diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index 73e706e..bd33fde 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -325,7 +325,7 @@ void gst_projectm_base_set_property(GObject *object, case PROP_IS_LIVE: g_free(settings->is_live); settings->is_live = g_strdup(g_value_get_string(value)); - g_object_set(G_OBJECT(glav), "is-live", settings->is_live, NULL); + g_object_set(G_OBJECT(glav), "pipeline-live", settings->is_live, NULL); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); diff --git a/src/renderbuffer.c b/src/renderbuffer.c index d95c44c..c6f4a13 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -86,22 +86,24 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; static void rb_handle_adaptive_fps_ema(RBRenderBuffer *state, const GstClockTime render_duration, const GstClockTime frame_duration) { - state->frame_counter++; + g_assert(state != NULL); + + state->ema_frame_counter++; // EMA smoothing: smoothed = alpha * x + (1 - alpha) * prev - state->smoothed_render_time = + state->ema_smoothed_render_time = gst_util_uint64_scale_int(render_duration, RB_EMA_ALPHA_N, RB_EMA_ALPHA_D) + - gst_util_uint64_scale_int(state->smoothed_render_time, + gst_util_uint64_scale_int(state->ema_smoothed_render_time, RB_EMA_ALPHA_D - RB_EMA_ALPHA_N, RB_EMA_ALPHA_D); - if (state->frame_counter >= RB_EMA_FPS_ADJUST_INTERVAL) { + if (state->ema_frame_counter >= RB_EMA_FPS_ADJUST_INTERVAL) { GstClockTime new_duration; - state->frame_counter = 0; + state->ema_frame_counter = 0; const GstClockTime upper_threshold = gst_util_uint64_scale_int( frame_duration, RB_EMA_FRAME_DURATION_TOLERANCE_UP_N, @@ -111,13 +113,13 @@ static void rb_handle_adaptive_fps_ema(RBRenderBuffer *state, frame_duration, RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N, RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D); - if (state->smoothed_render_time > upper_threshold) { + if (state->ema_smoothed_render_time > upper_threshold) { // rendering too slow, increase frame duration (drop FPS) new_duration = gst_util_uint64_scale_int( frame_duration, RB_EMA_FRAME_DURATION_INCREASE_N, RB_EMA_FRAME_DURATION_INCREASE_D); - } else if (state->smoothed_render_time < lower_threshold) { + } else if (state->ema_smoothed_render_time < lower_threshold) { // rendering fast enough, try to decrease frame duration (increase FPS) new_duration = gst_util_uint64_scale_int( @@ -152,8 +154,9 @@ static void rb_handle_adaptive_fps_ema(RBRenderBuffer *state, } } -static void rb_queue_gl_buffer_cleanup(const RBRenderBuffer *state, - GstBuffer *out) { +static void rb_queue_gl_buffer_cleanup(RBRenderBuffer *state, GstBuffer *out) { + g_assert(state != NULL); + g_assert(out != NULL); g_async_queue_push(state->buffer_cleanup_queue, out); } @@ -165,38 +168,35 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, const gboolean is_qos_enabled, const gboolean is_realtime) { + g_assert(state != NULL); + GST_DEBUG_CATEGORY_INIT(renderbuffer_debug, "renderbuffer", 0, "projectM visualizer plugin render buffer"); + // context config without ownership state->plugin = plugin; state->adjust_fps_func = adjust_fps_func; + // we'll get these later state->gl_context = NULL; state->src_pad = NULL; - state->render_thread = NULL; - g_atomic_int_set(&state->running, FALSE); + // never changed after init state->qos_enabled = is_qos_enabled; state->is_realtime = is_realtime; state->caps_frame_duration = caps_frame_duration; state->max_frame_duration = max_frame_duration; - state->last_insert_index = -1; - state->last_render_index = -1; - state->frame_counter = 0; - state->smoothed_render_time = 0; - state->buffer_push_queue_read_idx = -1; - state->buffer_push_queue_write_idx = -1; + // changed all the time + g_atomic_int_set(&state->running, FALSE); + // init render queue + state->render_thread = NULL; + state->render_write_idx = -1; + state->render_read_idx = -1; g_mutex_init(&state->slot_lock); - g_mutex_init(&state->buffer_push_queue_mutex); g_cond_init(&state->slot_available_cond); g_cond_init(&state->render_queued_cond); - g_cond_init(&state->buffer_push_queue_cond); - state->buffer_cleanup_queue = g_async_queue_new(); - for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { - state->buffer_push_queue[i] = NULL; - } for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { state->slots[i].state = RB_EMPTY; @@ -208,20 +208,45 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, state->slots[i].gl_fill_func = gl_fill_func; state->slots[i].in_audio = NULL; } + + // init EMA + state->ema_frame_counter = 0; + state->ema_smoothed_render_time = 0; + + // init push queue + state->push_thread = NULL; + state->push_queue_read_idx = -1; + state->push_queue_write_idx = -1; + g_mutex_init(&state->push_queue_mutex); + g_cond_init(&state->push_queue_cond); + for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + state->push_queue[i] = NULL; + } + + // init clean up queue + state->cleanup_thread = NULL; + state->buffer_cleanup_queue = g_async_queue_new(); } void rb_dispose_render_buffer(RBRenderBuffer *state) { + g_assert(state != NULL); + g_async_queue_unref(state->buffer_cleanup_queue); + g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); - g_cond_clear(&state->buffer_push_queue_cond); + g_cond_clear(&state->push_queue_cond); + g_mutex_clear(&state->slot_lock); - g_mutex_clear(&state->buffer_push_queue_mutex); + g_mutex_clear(&state->push_queue_mutex); } RBQueueResult rb_queue_render_task(RBQueueArgs *args) { + g_assert(args != NULL); RBRenderBuffer *state = args->render_buffer; + g_assert(state != NULL); + const gboolean wait_is_limited = args->max_wait != GST_CLOCK_TIME_NONE; const GstClockTime start = gst_util_get_timestamp(); GstClockTimeDiff used_wait = 0; @@ -229,18 +254,18 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { g_mutex_lock(&state->slot_lock); RBSlot *slot = NULL; - gint slot_index; + gint slot_index = 0; gboolean found_slot = FALSE; while (!found_slot) { // next slot to insert to - slot_index = (state->last_insert_index + 1) % NUM_RENDER_SLOTS; + slot_index = (state->render_write_idx + 1) % NUM_RENDER_SLOTS; slot = &state->slots[slot_index]; // jump over busy slot that's currently rendering if needed if (slot->state == RB_BUSY) { - slot_index = (state->last_insert_index + 2) % NUM_RENDER_SLOTS; + slot_index = (state->render_write_idx + 2) % NUM_RENDER_SLOTS; } // in case there is only one slot, it may still be busy @@ -249,7 +274,8 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { if (!found_slot) { if (wait_is_limited) { - const GstClockTimeDiff remaining_wait = args->max_wait - used_wait; + const GstClockTimeDiff remaining_wait = + (GstClockTimeDiff)args->max_wait - used_wait; // not waiting until the very last millisecond // to avoid exceeding time budget if (remaining_wait > MIN_FREE_SLOT_SCHEDULE_WAIT) { @@ -261,7 +287,7 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { g_cond_wait_until(&state->slot_available_cond, &state->slot_lock, deadline); - used_wait = (GstClockTimeDiff)gst_util_get_timestamp() - start; + used_wait = GST_CLOCK_DIFF(start, gst_util_get_timestamp()); } else { // not enough time left break; @@ -280,7 +306,7 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { return RB_TIMEOUT; } - state->last_insert_index = slot_index; + state->render_write_idx = slot_index; // evict if already in use and clear buffers const gboolean is_evicted = slot->state == RB_READY; @@ -316,7 +342,12 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { void rb_queue_render_task_log(RBQueueArgs *args) { + g_assert(args != NULL); + RBRenderBuffer *state = args->render_buffer; + + g_assert(state != NULL); + const GstClockTime start_ts = gst_util_get_timestamp(); const RBQueueResult result = rb_queue_render_task(args); @@ -365,6 +396,8 @@ gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, const GstClockTime running_time, const GstClockTime tolerance) { + g_assert(element != NULL); + if (latency == GST_CLOCK_TIME_NONE) { return FALSE; } @@ -394,6 +427,7 @@ gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, * @param buf GL buffer to release. */ static void gl_buffer_cleanup(GstGLContext *context, gpointer buf) { + (void)context; gst_buffer_unref(GST_BUFFER(buf)); } @@ -406,7 +440,9 @@ static void gl_buffer_cleanup(GstGLContext *context, gpointer buf) { */ static gpointer rb_cleanup_thread_func(gpointer user_data) { - const RBRenderBuffer *state = (RBRenderBuffer *)user_data; + RBRenderBuffer *state = (RBRenderBuffer *)user_data; + g_assert(state != NULL); + // consume gl buffers to dispatch to gl thread for cleanup while (g_atomic_int_get(&state->running)) { @@ -430,12 +466,13 @@ static gpointer rb_cleanup_thread_func(gpointer user_data) { */ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { // measure rendering for QoS - GstClockTime render_start = gst_util_get_timestamp(); + const GstClockTime render_start = gst_util_get_timestamp(); // Dispatch slot to GL thread gst_gl_context_thread_add(state->gl_context, slot->gl_fill_func, slot); - GstClockTime render_time = gst_util_get_timestamp() - render_start; + const GstClockTime render_time = + GST_CLOCK_DIFF(render_start, gst_util_get_timestamp()); // render took longer than the frame duration, this is a problem for // real-time rendering if it happens too often @@ -460,29 +497,54 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { * @param state The render buffer to use. * @param buffer The gl buffer to push. Takes ownership of the buffer. */ -static void rb_schedule_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { - g_mutex_lock(&state->buffer_push_queue_mutex); +static void rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { + g_assert(state != NULL); + g_assert(buffer != NULL); + + g_mutex_lock(&state->push_queue_mutex); // just write to the next position in the ring, no matter what - state->buffer_push_queue_write_idx = - (state->buffer_push_queue_write_idx + 1) % PUSH_QUEUE_MAX_SIZE; + state->push_queue_write_idx = + (state->push_queue_write_idx + 1) % PUSH_QUEUE_MAX_SIZE; - // dispose buffer, if any - if (state->buffer_push_queue[state->buffer_push_queue_write_idx] != NULL) { - rb_queue_gl_buffer_cleanup( - state, state->buffer_push_queue[state->buffer_push_queue_write_idx]); + // dispose of buffer, if any + if (state->push_queue[state->push_queue_write_idx] != NULL) { + rb_queue_gl_buffer_cleanup(state, + state->push_queue[state->push_queue_write_idx]); } // take the spot and signal that queue has changed - state->buffer_push_queue[state->buffer_push_queue_write_idx] = buffer; - g_cond_signal(&state->buffer_push_queue_cond); + state->push_queue[state->push_queue_write_idx] = buffer; + g_cond_signal(&state->push_queue_cond); + + g_mutex_unlock(&state->push_queue_mutex); +} + +/** + * Removes and disposes all queued buffers and resets queue state. + * + * @param state State to clear. + */ +static void rb_clear_push_queue(RBRenderBuffer *state) { + g_assert(state != NULL); + g_mutex_lock(&state->push_queue_mutex); + + // release buffers that are still queued before cleanup thread shuts down + for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + if (state->push_queue[i] != NULL) { + rb_queue_gl_buffer_cleanup(state, state->push_queue[i]); + state->push_queue[i] = NULL; + } + } + state->push_queue_read_idx = -1; + state->push_queue_write_idx = -1; - g_mutex_unlock(&state->buffer_push_queue_mutex); + g_mutex_unlock(&state->push_queue_mutex); } /** * Pushes gl buffers for real-time rendering only. - * Consume buffers to push and wait until it's time to push. + * Consume buffers to push and wait until it's PTS time to push. * * @param user_data Render buffer to use. * @return NULL @@ -490,97 +552,83 @@ static void rb_schedule_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { static gpointer rb_push_thread_func(gpointer user_data) { RBRenderBuffer *state = (RBRenderBuffer *)user_data; + g_assert(state != NULL); - // buffers are in PTS order, wait until it's time to push - - GstClockTimeDiff used_wait; - GstClockTimeDiff frame_wait; - GstClockTime wait_start; - - g_mutex_lock(&state->buffer_push_queue_mutex); + g_mutex_lock(&state->push_queue_mutex); while (g_atomic_int_get(&state->running)) { - state->buffer_push_queue_read_idx = - (state->buffer_push_queue_read_idx + 1) % PUSH_QUEUE_MAX_SIZE; + state->push_queue_read_idx = + (state->push_queue_read_idx + 1) % PUSH_QUEUE_MAX_SIZE; // consume gl buffer to push - if (state->buffer_push_queue[state->buffer_push_queue_read_idx] == NULL) { + if (state->push_queue[state->push_queue_read_idx] == NULL) { // no buffer to push, wait for one - state->buffer_push_queue_read_idx = state->buffer_push_queue_write_idx; - g_cond_wait(&state->buffer_push_queue_cond, - &state->buffer_push_queue_mutex); + state->push_queue_read_idx = state->push_queue_write_idx; + g_cond_wait(&state->push_queue_cond, &state->push_queue_mutex); continue; } // the buffer to push (for now, we may still lose it) - GstBuffer *outbuf = - state->buffer_push_queue[state->buffer_push_queue_read_idx]; + GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; + + gboolean abort = FALSE; // determine when it's time to push GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); + if (clock) { + const GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); - const GstClockTime pts = GST_BUFFER_PTS(outbuf); - const GstClockTime abs_time = pts + base_time; + const GstClockTime pts = GST_BUFFER_PTS(outbuf); + const GstClockTime abs_time = pts + base_time; - wait_start = gst_clock_get_time(clock); - frame_wait = GST_CLOCK_DIFF(wait_start, abs_time); - used_wait = 0; - gboolean abort = FALSE; + GstClockTimeDiff remaining_wait = + GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); - // interruptable wait, continues when ring buffer read pointer is overtaken - // by write pointer while waiting - while (used_wait < frame_wait) { + // interruptable wait, continues when ring buffer read pointer is + // overtaken by write pointer while waiting + while (remaining_wait > GST_USECOND) { + if (!g_atomic_int_get(&state->running)) { + abort = TRUE; + break; + } - abort = !g_atomic_int_get(&state->running); - if (abort) { - break; - } + // wait until it's time to push or another buffer is queued + // (microseconds!) + g_cond_wait_until(&state->push_queue_cond, &state->push_queue_mutex, + g_get_monotonic_time() + remaining_wait / 1000); - // wait until it's time to push or another buffer is queued - // (microseconds!) - g_cond_wait_until(&state->buffer_push_queue_cond, - &state->buffer_push_queue_mutex, - g_get_real_time() + (frame_wait - used_wait) / 1000); - used_wait = (GstClockTimeDiff)gst_clock_get_time(clock) - wait_start; - - // were we overtaken ? - if (state->buffer_push_queue[state->buffer_push_queue_read_idx] != - outbuf) { - abort = TRUE; - break; + // have we been overtaken ? + if (state->push_queue[state->push_queue_read_idx] != outbuf) { + abort = TRUE; + break; + } + remaining_wait = GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); } + gst_object_unref(clock); } - gst_object_unref(clock); if (abort) { continue; } // now we own the buffer to push - state->buffer_push_queue[state->buffer_push_queue_read_idx] = NULL; + state->push_queue[state->push_queue_read_idx] = NULL; - g_mutex_unlock(&state->buffer_push_queue_mutex); + g_mutex_unlock(&state->push_queue_mutex); // push buffer downstream const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); + if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } - g_mutex_lock(&state->buffer_push_queue_mutex); - } - // release buffers that are still queued before cleanup thread shuts down - for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { - if (state->buffer_push_queue[i] != NULL) { - rb_queue_gl_buffer_cleanup(state, state->buffer_push_queue[i]); - state->buffer_push_queue[i] = NULL; - } + g_mutex_lock(&state->push_queue_mutex); } - g_mutex_unlock(&state->buffer_push_queue_mutex); + g_mutex_unlock(&state->push_queue_mutex); return NULL; } @@ -596,9 +644,11 @@ static gpointer rb_push_thread_func(gpointer user_data) { * @param frame_duration Frame duration. * @return TRUE if the buffer was pushed successfully. */ -GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, - GstClockTime pts, - GstClockTime frame_duration) { +static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, + GstBuffer *outbuf, + const GstClockTime pts, + const GstClockTime frame_duration) { + g_assert(state != NULL); if (gst_buffer_get_size(outbuf) == 0) { GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); @@ -615,7 +665,7 @@ GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, // for real-time, we need to wait until it's time to push the buffer // dispatch the wait and push to another thread to keep rendering as fast // as we can (and get audio buffers) - rb_schedule_push_buffer(state, outbuf); + rb_queue_push_buffer(state, outbuf); ret = GST_FLOW_OK; } else { // push buffer downstream directly for offline rendering, avoid scheduling @@ -632,12 +682,14 @@ GstFlowReturn rb_handle_send_buffer(RBRenderBuffer *state, GstBuffer *outbuf, } /** - * Reset buffer references, set slot state to RB_EMPTY and signal. + * Reset render buffer references, set slot state to RB_EMPTY and signal. * * @param state Render buffer to use. * @param slot Slot to release. */ static void rb_release_slot(RBRenderBuffer *state, RBSlot *slot) { + g_assert(state != NULL); + // Lock and reset slot data g_mutex_lock(&state->slot_lock); @@ -654,6 +706,8 @@ static void rb_release_slot(RBRenderBuffer *state, RBSlot *slot) { GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, GstClockTime pts, GstClockTime frame_duration) { + g_assert(state != NULL); + // Lock and reset slot data g_mutex_lock(&state->slot_lock); @@ -669,7 +723,7 @@ GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, rb_render_slot(state, slot); GstFlowReturn ret = - rb_handle_send_buffer(state, slot->out_buf, pts, frame_duration); + rb_handle_push_buffer(state, slot->out_buf, pts, frame_duration); // reset slot slot->in_audio = NULL; @@ -682,6 +736,40 @@ GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, return ret; } +/** + * Clears all render slots, releases all buffers currently held, resets the + * render queue state and EMA state. + * + * @param state The render buffer to clear. + */ +static void rb_clear_slots(RBRenderBuffer *state) { + g_assert(state != NULL); + + g_mutex_lock(&state->slot_lock); + + // clean up queue and state + for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { + if (state->slots[i].state == RB_READY) { + if (state->slots[i].in_audio) { + gst_buffer_unref(state->slots[i].in_audio); + state->slots[i].in_audio = NULL; + } + if (state->slots[i].out_buf) { + rb_queue_gl_buffer_cleanup(state, state->slots[i].out_buf); + state->slots[i].out_buf = NULL; + } + state->slots[i].state = RB_EMPTY; + } + } + + state->render_write_idx = -1; + state->render_read_idx = -1; + state->ema_frame_counter = 0; + state->ema_smoothed_render_time = 0; + + g_mutex_unlock(&state->slot_lock); +} + /** * Render thread main worker function. * @@ -691,6 +779,8 @@ GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, static gpointer rb_render_thread_func(gpointer user_data) { RBRenderBuffer *state = (RBRenderBuffer *)user_data; + g_assert(state != NULL); + #if NUM_RENDER_SLOTS > 2 GstClockTime last_pts = GST_CLOCK_TIME_NONE; #endif @@ -702,12 +792,12 @@ static gpointer rb_render_thread_func(gpointer user_data) { // first find a slot with data that's ready to render gboolean found_slot = FALSE; RBSlot *slot = NULL; - gint render_index; + gint render_index = 0; g_mutex_lock(&state->slot_lock); while (!found_slot) { - render_index = (state->last_render_index + 1) % NUM_RENDER_SLOTS; + render_index = (state->render_read_idx + 1) % NUM_RENDER_SLOTS; slot = &state->slots[render_index]; @@ -743,7 +833,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { } // update read maker - state->last_render_index = render_index; + state->render_read_idx = render_index; #if NUM_RENDER_SLOTS > 2 last_pts = slot->pts; #endif @@ -758,17 +848,22 @@ static gpointer rb_render_thread_func(gpointer user_data) { if (state->is_realtime) { GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); if (clock) { + // get current running time / pts GstClockTime base_time = gst_element_get_base_time(GST_ELEMENT(state->plugin)); - GstClockTime now = gst_clock_get_time(clock) - base_time; + + const GstClockTime running_time = + GST_CLOCK_DIFF(base_time, gst_clock_get_time(clock)); // wait if we're too far ahead - if (now < (GstClockTimeDiff)slot->pts - slot->frame_duration) { - gst_clock_id_wait( - gst_clock_new_single_shot_id(clock, base_time + slot->pts), NULL); + if (running_time + slot->frame_duration < slot->pts) { + GstClockID id = + gst_clock_new_single_shot_id(clock, base_time + slot->pts); + gst_clock_id_wait(id, NULL); + gst_clock_id_unref(id); } + gst_object_unref(clock); } - gst_object_unref(clock); } // perform gl rendering @@ -787,7 +882,7 @@ static gpointer rb_render_thread_func(gpointer user_data) { rb_release_slot(state, slot); // send out buffer downstream - if (rb_handle_send_buffer(state, outbuf, pts, frame_duration) == + if (rb_handle_push_buffer(state, outbuf, pts, frame_duration) == GST_FLOW_OK) { // process rendering fps QoS in case frame was pushed @@ -806,27 +901,34 @@ static gpointer rb_render_thread_func(gpointer user_data) { } } - g_mutex_lock(&state->slot_lock); - for (guint i = 0; i < NUM_RENDER_SLOTS; i++) { - if (state->slots[i].state == RB_READY) { - if (state->slots[i].in_audio) { - gst_buffer_unref(state->slots[i].in_audio); - state->slots[i].in_audio = NULL; - } - if (state->slots[i].out_buf) { - rb_queue_gl_buffer_cleanup(state, state->slots[i].out_buf); - state->slots[i].out_buf = NULL; - } - state->slots[i].state = RB_EMPTY; - } + return NULL; +} + +static void rb_clear_cleanup_queue(RBRenderBuffer *state) { + g_assert(state != NULL); + // make sure all gl buffers are released + gpointer item; + while ((item = g_async_queue_try_pop(state->buffer_cleanup_queue)) != NULL) { + gst_gl_context_thread_add(state->gl_context, gl_buffer_cleanup, item); } - g_mutex_unlock(&state->slot_lock); +} - return NULL; +/** + * Clears all queues. + * + * @param state Render buffer to clear. + */ +void rb_clear(RBRenderBuffer *state) { + g_assert(state != NULL); + + rb_clear_slots(state); + rb_clear_push_queue(state); + rb_clear_cleanup_queue(state); } void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, GstPad *src_pad) { + g_assert(state != NULL); state->gl_context = gl_context; state->src_pad = src_pad; g_atomic_int_set(&state->running, TRUE); @@ -845,22 +947,34 @@ void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, } void rb_stop(RBRenderBuffer *state) { + g_assert(state != NULL); g_atomic_int_set(&state->running, FALSE); // threads are not needed for offline rendering if (state->is_realtime) { + // wake up render thread to signal loop exit g_mutex_lock(&state->slot_lock); g_cond_broadcast(&state->render_queued_cond); g_mutex_unlock(&state->slot_lock); + // wait for render thread to exit g_thread_join(state->render_thread); + state->render_thread = NULL; + + // signal wake up push thread to singal loop exit + g_mutex_lock(&state->push_queue_mutex); + g_cond_broadcast(&state->push_queue_cond); + g_mutex_unlock(&state->push_queue_mutex); - rb_schedule_push_buffer(state, NULL); + // wait for push thread to exit g_thread_join(state->push_thread); + state->push_thread = NULL; + // signal and wait for cleanup thread to exit g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); g_thread_join(state->cleanup_thread); + state->cleanup_thread = NULL; } state->gl_context = NULL; @@ -870,6 +984,8 @@ void rb_stop(RBRenderBuffer *state) { void rb_set_caps_frame_duration(RBRenderBuffer *state, const GstClockTime caps_frame_duration) { + g_assert(state != NULL); + g_mutex_lock(&state->slot_lock); state->caps_frame_duration = caps_frame_duration; g_mutex_unlock(&state->slot_lock); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 3f8da1c..39d4007 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -1,17 +1,32 @@ /* - * A ring buffer based render buffer to allow offloading of - * rendering tasks from the plugin chain function. The ring buffer consists of - * a limited number of rendering slots. + * Utility to allow offloading of rendering tasks from the plugin chain + * function. + * + * Functionality provided: + * - A ring buffer based rendering task queue with a limited number of rendering + * slots. It is being consumed by a dedicated thread (render thread) to + * dispatch rendering to the GL thread. + * + * - A ring buffer based GL buffer queue to schedule GL buffers to be pushed + * downstream at presentation time. It is being consumed by a dedicated thread + * (push thread). + * + * - An async queue to dispose of dropped GL buffers. It is being consumed by + * a dedicated thread (cleanup thread) to dispatch GL buffer cleanup to the GL + * thread. * * For offline pipelines only: * - * - A blocking call is used for rendering, bypasses queuing. + * - A blocking call is used for rendering, bypassing queuing. GL buffers are + * never dropped. + * + * --- * * For real-time pipelines only: * - * The buffer provides queueing for audio buffers to be rendered to video - * frames. It uses a bound-wait-on-full approach to avoid dropping frames when - * rendering duration exceeds the frame duration of the current fps: + * - The buffer provides queueing for audio buffers to be rendered to video + * frames. It uses a bound-wait-on-full approach to avoid dropping frames when + * rendering duration exceeds the frame duration of the current fps: * * - In case a free slot is available queue * immediately and return (async rendering). @@ -30,22 +45,22 @@ * * --- * - * - If the render duration exceeds the fps *sometimes*, subsequent - * faster-than-real-time rendered frames (if any) compensate for the small - * lag, frames are dropped or eventually QoS events from the downstream - * sink will re-sync with the pipeline clock. + * - If the render duration exceeds the fps *sometimes*, subsequent + * faster-than-real-time rendered frames (if any) compensate for the small + * lag, frames are dropped or eventually QoS events from the downstream + * sink will re-sync with the pipeline clock. * - * - If the render duration exceeds the fps *most of the time*, an Exponential - * Moving Average (EMA) based algorithm instructs the plugin to reduce fps. - * EMA will also recover fps when render performance increases again. + * - If the render duration exceeds the fps *most of the time*, an Exponential + * Moving Average (EMA) based algorithm instructs the plugin to reduce fps. + * EMA will also recover fps when render performance increases again. * - * - Buffers that completed rendering are scheduled to be pushed to the source - * pad at PTS. A ring buffer queues buffers for pushing. The buffer acts like - * a blocking queue with scheduling wait. A separate worker thread consumes - * the ring buffer and waits for reaching the PTS of the current buffer. The - * wait is interruptable, in case the read pointer is overtaken by the write - * pointer, waiting for PTS is aborted and the next frame is scheduled - * immediately. + * - GL buffers that completed rendering are scheduled to be pushed to the + * source pad at presentation time (PTS). The queue implemented as a ring + * buffer that acts like a blocking queue with scheduling wait. A separate + * worker thread consumes the ring buffer and waits for reaching the PTS of + * the current GL buffer. The wait is interruptable, in case the read pointer + * is overtaken by the write pointer, waiting for PTS is aborted and the next + * frame is scheduled immediately. */ #ifndef __RENDERBUFFER_H__ @@ -233,27 +248,17 @@ typedef struct { /** * Ring buffer to schedule gl buffers for pushing. */ - GstBuffer *buffer_push_queue[PUSH_QUEUE_MAX_SIZE]; - - /** - * Push ring buffer write position. - */ - gint buffer_push_queue_write_idx; - - /** - * Push ring buffer read position. - */ - gint buffer_push_queue_read_idx; + GstBuffer *push_queue[PUSH_QUEUE_MAX_SIZE]; /** * Mutex for push ring buffer. */ - GMutex buffer_push_queue_mutex; + GMutex push_queue_mutex; /** - * Condition as interruptable scheduling clock for push ring buffer. + * Condition signaled when a buffer has been queued. */ - GCond buffer_push_queue_cond; + GCond push_queue_cond; /** * Queue to dispose of dropped gl buffers. @@ -274,7 +279,7 @@ typedef struct { // -------------------------------------------------------------- /** - * TRUE if render thread is currently running. + * TRUE if rendering is currently running. */ gboolean running; @@ -315,31 +320,44 @@ typedef struct { */ RBSlot slots[NUM_RENDER_SLOTS]; - // only used by the calling thread (chain function) + // only used by the calling thread (chain function) / clean up // -------------------------------------------------------------- /** * Last index that data was inserted at (insertion pointer). */ - gint last_insert_index; + gint render_write_idx; - // only used by the render thread + // only used by the render thread / clean up // -------------------------------------------------------------- /** * Last index that data was rendered from (read pointer). */ - gint last_render_index; + gint render_read_idx; /** * EMA frame counter. */ - guint frame_counter; + guint ema_frame_counter; /** * EMA running average. */ - guint64 smoothed_render_time; + guint64 ema_smoothed_render_time; + + // concurrent access, protected by push_queue_mutex + // -------------------------------------------------------------- + + /** + * Push ring buffer write position. + */ + gint push_queue_write_idx; + + /** + * Push ring buffer read position. + */ + gint push_queue_read_idx; } RBRenderBuffer; @@ -451,6 +469,13 @@ static gboolean rb_is_render_too_late(GstElement *element, GstClockTime latency, GstClockTime running_time, GstClockTime tolerance); +/** + * Clears all queues. + * + * @param state Render buffer to clear. + */ +void rb_clear(RBRenderBuffer *state); + /** * Start render loop. * From 5d7b34b011bd5f8c74eaee328823fce501c29cf2 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sun, 12 Oct 2025 03:12:22 -0500 Subject: [PATCH 16/32] use blocking gl buffer push queue to throttle render loop --- src/gstglbaseaudiovisualizer.c | 1 + src/renderbuffer.c | 128 +++++++++++++++------------------ src/renderbuffer.h | 24 ++++--- 3 files changed, 74 insertions(+), 79 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 900acab..c5194a4 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -524,6 +524,7 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( GstAVRenderParams *render_data) { + (void)render_data; return TRUE; } diff --git a/src/renderbuffer.c b/src/renderbuffer.c index c6f4a13..ee576d8 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -73,6 +73,15 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; #define MIN_FREE_SLOT_SCHEDULE_WAIT (GST_MSECOND * 1) #endif +/** + * Tolerance / minimal wait time for scheduling a timed wait before pushing a + * buffer. If the calculated wait time is less than this value, the wait will be + * skipped. + */ +#ifndef MIN_PUSH_SCHEDULE_WAIT +#define MIN_PUSH_SCHEDULE_WAIT (GST_USECOND * 20) +#endif + /** * Exponential Moving Average (EMA)-based adaptive frame duration (fps) * adjustment. Determines desired frame duration change based on the @@ -219,6 +228,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, state->push_queue_write_idx = -1; g_mutex_init(&state->push_queue_mutex); g_cond_init(&state->push_queue_cond); + g_cond_init(&state->push_queue_free_cond); for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { state->push_queue[i] = NULL; } @@ -236,6 +246,7 @@ void rb_dispose_render_buffer(RBRenderBuffer *state) { g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); g_cond_clear(&state->push_queue_cond); + g_cond_clear(&state->push_queue_free_cond); g_mutex_clear(&state->slot_lock); g_mutex_clear(&state->push_queue_mutex); @@ -491,33 +502,43 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { /** * Schedule a rendered gl buffer for pushing downstream. * The buffer will not be pushed until it's PTS time is reached. - * Subsequent queued buffers will override existing buffers if they are not due - * for pushing in time. + * This call will block until the buffer can be schduled or + * the render buffer is stopped. * * @param state The render buffer to use. * @param buffer The gl buffer to push. Takes ownership of the buffer. + * + * @return TRUE if the buffer was scheduled successfully, FALSE in case the + * buffer was stopped. */ -static void rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { +static gboolean rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { g_assert(state != NULL); g_assert(buffer != NULL); g_mutex_lock(&state->push_queue_mutex); - // just write to the next position in the ring, no matter what + // write to the next position in the ring state->push_queue_write_idx = (state->push_queue_write_idx + 1) % PUSH_QUEUE_MAX_SIZE; - // dispose of buffer, if any - if (state->push_queue[state->push_queue_write_idx] != NULL) { - rb_queue_gl_buffer_cleanup(state, - state->push_queue[state->push_queue_write_idx]); + gboolean result = TRUE; + while (state->push_queue[state->push_queue_write_idx] != NULL) { + g_cond_wait(&state->push_queue_free_cond, &state->push_queue_mutex); + if (!g_atomic_int_get(&state->running)) { + result = FALSE; + break; + } } - // take the spot and signal that queue has changed - state->push_queue[state->push_queue_write_idx] = buffer; - g_cond_signal(&state->push_queue_cond); + if (result) { + // take the spot and signal that queue has changed + state->push_queue[state->push_queue_write_idx] = buffer; + g_cond_signal(&state->push_queue_cond); + } g_mutex_unlock(&state->push_queue_mutex); + + return result; } /** @@ -562,17 +583,22 @@ static gpointer rb_push_thread_func(gpointer user_data) { (state->push_queue_read_idx + 1) % PUSH_QUEUE_MAX_SIZE; // consume gl buffer to push - if (state->push_queue[state->push_queue_read_idx] == NULL) { + gboolean stop = FALSE; + while (state->push_queue[state->push_queue_read_idx] == NULL) { // no buffer to push, wait for one - state->push_queue_read_idx = state->push_queue_write_idx; g_cond_wait(&state->push_queue_cond, &state->push_queue_mutex); - continue; + if (!g_atomic_int_get(&state->running)) { + stop = TRUE; + break; + } } - // the buffer to push (for now, we may still lose it) - GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; + if (stop) { + break; + } - gboolean abort = FALSE; + // found a buffer to push + GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; // determine when it's time to push GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); @@ -586,36 +612,18 @@ static gpointer rb_push_thread_func(gpointer user_data) { GstClockTimeDiff remaining_wait = GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); - // interruptable wait, continues when ring buffer read pointer is - // overtaken by write pointer while waiting - while (remaining_wait > GST_USECOND) { - if (!g_atomic_int_get(&state->running)) { - abort = TRUE; - break; - } - - // wait until it's time to push or another buffer is queued - // (microseconds!) - g_cond_wait_until(&state->push_queue_cond, &state->push_queue_mutex, - g_get_monotonic_time() + remaining_wait / 1000); - - // have we been overtaken ? - if (state->push_queue[state->push_queue_read_idx] != outbuf) { - abort = TRUE; - break; - } - remaining_wait = GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); + if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { + GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); + gst_clock_id_wait(clock_id, NULL); + gst_clock_id_unref(clock_id); } - gst_object_unref(clock); - } - if (abort) { - continue; + gst_object_unref(clock); } // now we own the buffer to push state->push_queue[state->push_queue_read_idx] = NULL; - + g_cond_signal(&state->push_queue_free_cond); g_mutex_unlock(&state->push_queue_mutex); // push buffer downstream @@ -654,7 +662,6 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); rb_queue_gl_buffer_cleanup(state, outbuf); } else { - // populate timestamps after rendering so they can't be changed by accident GST_BUFFER_PTS(outbuf) = pts; GST_BUFFER_DTS(outbuf) = pts; @@ -663,10 +670,14 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, GstFlowReturn ret; if (state->is_realtime) { // for real-time, we need to wait until it's time to push the buffer - // dispatch the wait and push to another thread to keep rendering as fast - // as we can (and get audio buffers) - rb_queue_push_buffer(state, outbuf); - ret = GST_FLOW_OK; + // dispatch to queue may block until capacity is available + gboolean result = rb_queue_push_buffer(state, outbuf); + if (result) { + ret = GST_FLOW_OK; + } else { + rb_queue_gl_buffer_cleanup(state, outbuf); + ret = GST_FLOW_ERROR; + } } else { // push buffer downstream directly for offline rendering, avoid scheduling // overhead @@ -844,30 +855,8 @@ static gpointer rb_render_thread_func(gpointer user_data) { g_mutex_unlock(&state->slot_lock); - // throttle rendering - if (state->is_realtime) { - GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - // get current running time / pts - GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - - const GstClockTime running_time = - GST_CLOCK_DIFF(base_time, gst_clock_get_time(clock)); - - // wait if we're too far ahead - if (running_time + slot->frame_duration < slot->pts) { - GstClockID id = - gst_clock_new_single_shot_id(clock, base_time + slot->pts); - gst_clock_id_wait(id, NULL); - gst_clock_id_unref(id); - } - gst_object_unref(clock); - } - } - // perform gl rendering - GstClockTime render_time = rb_render_slot(state, slot); + const GstClockTime render_time = rb_render_slot(state, slot); // copy params to locals vars to release the slot GstBuffer *audio_buffer = slot->in_audio; @@ -965,6 +954,7 @@ void rb_stop(RBRenderBuffer *state) { // signal wake up push thread to singal loop exit g_mutex_lock(&state->push_queue_mutex); g_cond_broadcast(&state->push_queue_cond); + g_cond_broadcast(&state->push_queue_free_cond); g_mutex_unlock(&state->push_queue_mutex); // wait for push thread to exit diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 39d4007..740bd6e 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -7,7 +7,7 @@ * slots. It is being consumed by a dedicated thread (render thread) to * dispatch rendering to the GL thread. * - * - A ring buffer based GL buffer queue to schedule GL buffers to be pushed + * - A ring buffer based queue to schedule GL buffers to be pushed * downstream at presentation time. It is being consumed by a dedicated thread * (push thread). * @@ -56,11 +56,10 @@ * * - GL buffers that completed rendering are scheduled to be pushed to the * source pad at presentation time (PTS). The queue implemented as a ring - * buffer that acts like a blocking queue with scheduling wait. A separate - * worker thread consumes the ring buffer and waits for reaching the PTS of - * the current GL buffer. The wait is interruptable, in case the read pointer - * is overtaken by the write pointer, waiting for PTS is aborted and the next - * frame is scheduled immediately. + * buffer that blocks on insert in case it is at capacity until the buffer + * can be scheduled. The render loop is throttled by waiting on a free slot. + * A separate worker thread consumes the ring buffer and waits for reaching + * the PTS of the current GL buffer before pushing it downstream. */ #ifndef __RENDERBUFFER_H__ @@ -90,12 +89,11 @@ G_BEGIN_DECLS /** * Max number of gl frame buffers waiting in a scheduled state to be pushed. - * Capacity should be low. Frames will be dropped if newer frame are queued, - * last frame wins. Render loop is capped for real-time, there should not be a - * lot of buffers waiting. + * Capacity should be low. Queuing call will block when capacity is reached and + * throttle the render loop, frames are never dropped. */ #ifndef PUSH_QUEUE_MAX_SIZE -#define PUSH_QUEUE_MAX_SIZE 3 +#define PUSH_QUEUE_MAX_SIZE 2 #endif /** @@ -260,6 +258,12 @@ typedef struct { */ GCond push_queue_cond; + /** + * Condition signaled when a buffer has been pushed + * and a slot if free. + */ + GCond push_queue_free_cond; + /** * Queue to dispose of dropped gl buffers. */ From 5f6b21c74013fe2377502b79da2dbe7f110cf6d3 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sun, 12 Oct 2025 19:49:20 -0500 Subject: [PATCH 17/32] allow disabling push queue at compile time --- src/renderbuffer.c | 79 +++++++++++++++++++++++++++++++++++++--------- src/renderbuffer.h | 30 +++++++++++------- 2 files changed, 83 insertions(+), 26 deletions(-) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index ee576d8..e85cda4 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -25,42 +25,57 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; */ #ifndef RB_EMA_ALPHA_N #define RB_EMA_ALPHA_N 1 +#endif + +#ifndef RB_EMA_ALPHA_D #define RB_EMA_ALPHA_D 5 #endif /** - * Increase frame duration (slow down fps) in case of detected lag. + * EMA increase frame duration (slow down fps) in case of detected lag. * +20% */ #ifndef RB_EMA_FRAME_DURATION_INCREASE_N #define RB_EMA_FRAME_DURATION_INCREASE_N 12 +#endif + +#ifndef RB_EMA_FRAME_DURATION_INCREASE_D #define RB_EMA_FRAME_DURATION_INCREASE_D 10 #endif /** - * Decrease frame duration (speed up fps) in case rendering performance + * EMA decrease frame duration (speed up fps) in case rendering performance * recovers. -5% */ #ifndef RB_EMA_FRAME_DURATION_DECREASE_N #define RB_EMA_FRAME_DURATION_DECREASE_N 95 +#endif + +#ifndef RB_EMA_FRAME_DURATION_DECREASE_D #define RB_EMA_FRAME_DURATION_DECREASE_D 100 #endif /** - * Tolerance for being too slow. + * EMA tolerance for being too slow. * Allow render time up to 1.1x */ #ifndef RB_EMA_FRAME_DURATION_TOLERANCE_UP_N #define RB_EMA_FRAME_DURATION_TOLERANCE_UP_N 110 +#endif + +#ifndef RB_EMA_FRAME_DURATION_TOLERANCE_UP_D #define RB_EMA_FRAME_DURATION_TOLERANCE_UP_D 100 #endif /** - * Tolerance for being too fast. + * EMA tolerance for being too fast. * allow render time as low as 0.9x */ #ifndef RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N #define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N 95 +#endif + +#ifndef RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D #define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_D 100 #endif @@ -229,7 +244,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, g_mutex_init(&state->push_queue_mutex); g_cond_init(&state->push_queue_cond); g_cond_init(&state->push_queue_free_cond); - for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { state->push_queue[i] = NULL; } @@ -511,6 +526,7 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { * @return TRUE if the buffer was scheduled successfully, FALSE in case the * buffer was stopped. */ +#if PUSH_QUEUE_SIZE > 0 static gboolean rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { g_assert(state != NULL); g_assert(buffer != NULL); @@ -519,7 +535,7 @@ static gboolean rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { // write to the next position in the ring state->push_queue_write_idx = - (state->push_queue_write_idx + 1) % PUSH_QUEUE_MAX_SIZE; + (state->push_queue_write_idx + 1) % PUSH_QUEUE_SIZE; gboolean result = TRUE; while (state->push_queue[state->push_queue_write_idx] != NULL) { @@ -540,18 +556,20 @@ static gboolean rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { return result; } +#endif /** * Removes and disposes all queued buffers and resets queue state. * * @param state State to clear. */ +#if PUSH_QUEUE_SIZE > 0 static void rb_clear_push_queue(RBRenderBuffer *state) { g_assert(state != NULL); g_mutex_lock(&state->push_queue_mutex); // release buffers that are still queued before cleanup thread shuts down - for (guint i = 0; i < PUSH_QUEUE_MAX_SIZE; i++) { + for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { if (state->push_queue[i] != NULL) { rb_queue_gl_buffer_cleanup(state, state->push_queue[i]); state->push_queue[i] = NULL; @@ -562,6 +580,7 @@ static void rb_clear_push_queue(RBRenderBuffer *state) { g_mutex_unlock(&state->push_queue_mutex); } +#endif /** * Pushes gl buffers for real-time rendering only. @@ -570,6 +589,7 @@ static void rb_clear_push_queue(RBRenderBuffer *state) { * @param user_data Render buffer to use. * @return NULL */ +#if PUSH_QUEUE_SIZE > 0 static gpointer rb_push_thread_func(gpointer user_data) { RBRenderBuffer *state = (RBRenderBuffer *)user_data; @@ -580,7 +600,7 @@ static gpointer rb_push_thread_func(gpointer user_data) { while (g_atomic_int_get(&state->running)) { state->push_queue_read_idx = - (state->push_queue_read_idx + 1) % PUSH_QUEUE_MAX_SIZE; + (state->push_queue_read_idx + 1) % PUSH_QUEUE_SIZE; // consume gl buffer to push gboolean stop = FALSE; @@ -640,11 +660,13 @@ static gpointer rb_push_thread_func(gpointer user_data) { return NULL; } +#endif /** * Send a video buffer to the source pad downstream. * Buffer is checked and timestamps are populated before sending. - * Push is blocking for offline rendering, and queued for real-time rendering. + * Push is blocking for offline rendering, and for real-time rendering queued if + * capacity is available, otherwise blocking. * * @param state Render buffer to use. * @param outbuf Video buffer to send downstream (takes ownership). @@ -669,6 +691,7 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, GstFlowReturn ret; if (state->is_realtime) { +#if PUSH_QUEUE_SIZE > 0 // for real-time, we need to wait until it's time to push the buffer // dispatch to queue may block until capacity is available gboolean result = rb_queue_push_buffer(state, outbuf); @@ -679,13 +702,32 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, ret = GST_FLOW_ERROR; } } else { - // push buffer downstream directly for offline rendering, avoid scheduling - // overhead +#else + // blocking wait until buffer has been pushed in time + // then push directly + GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + if (clock) { + const GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + + const GstClockTime abs_time = pts + base_time; + + GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); + gst_clock_id_wait(clock_id, NULL); + gst_clock_id_unref(clock_id); + } + gst_object_unref(clock); + } +#endif + // push buffer downstream directly for offline rendering or if + // queuing is disabled ret = gst_pad_push(state->src_pad, outbuf); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } +#if PUSH_QUEUE_SIZE > 0 } +#endif return ret; } @@ -818,8 +860,8 @@ static gpointer rb_render_thread_func(gpointer user_data) { if (slot->state == RB_READY #if NUM_RENDER_SLOTS > 2 // wontfix: segment events would need to be handled for this check to - // work right otherwise last_pts is not reset when the pts changes. - // if this is ever desired, each queued frame should have an + // work right otherwise last_pts is not reset when the pts offset + // changes. If this is ever desired, each queued frame should have an // incrementing id field to use for this check // check if next frame is already outdated, may happen if write @@ -871,6 +913,8 @@ static gpointer rb_render_thread_func(gpointer user_data) { rb_release_slot(state, slot); // send out buffer downstream + // call will block if rendering is running ahead + // and throttle render loop if (rb_handle_push_buffer(state, outbuf, pts, frame_duration) == GST_FLOW_OK) { @@ -911,7 +955,9 @@ void rb_clear(RBRenderBuffer *state) { g_assert(state != NULL); rb_clear_slots(state); +#if PUSH_QUEUE_SIZE > 0 rb_clear_push_queue(state); +#endif rb_clear_cleanup_queue(state); } @@ -926,8 +972,10 @@ void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, if (state->is_realtime) { state->render_thread = g_thread_new("rb-render-thread", rb_render_thread_func, state); +#if PUSH_QUEUE_SIZE > 0 state->push_thread = g_thread_new("rb-push-thread", rb_push_thread_func, state); +#endif state->cleanup_thread = g_thread_new("rb-cleanup-thread", rb_cleanup_thread_func, state); } @@ -951,15 +999,16 @@ void rb_stop(RBRenderBuffer *state) { g_thread_join(state->render_thread); state->render_thread = NULL; - // signal wake up push thread to singal loop exit +#if PUSH_QUEUE_SIZE > 0 + // signal wake up push thread to signal loop exit g_mutex_lock(&state->push_queue_mutex); g_cond_broadcast(&state->push_queue_cond); g_cond_broadcast(&state->push_queue_free_cond); g_mutex_unlock(&state->push_queue_mutex); - // wait for push thread to exit g_thread_join(state->push_thread); state->push_thread = NULL; +#endif // signal and wait for cleanup thread to exit g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 740bd6e..be5b23a 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -15,6 +15,8 @@ * a dedicated thread (cleanup thread) to dispatch GL buffer cleanup to the GL * thread. * + * --- + * * For offline pipelines only: * * - A blocking call is used for rendering, bypassing queuing. GL buffers are @@ -43,12 +45,10 @@ * with the current frame (evicted), meaning the previous frame is being * dropped as it is too late. * - * --- * * - If the render duration exceeds the fps *sometimes*, subsequent * faster-than-real-time rendered frames (if any) compensate for the small - * lag, frames are dropped or eventually QoS events from the downstream - * sink will re-sync with the pipeline clock. + * lag, or frames are dropped. * * - If the render duration exceeds the fps *most of the time*, an Exponential * Moving Average (EMA) based algorithm instructs the plugin to reduce fps. @@ -76,12 +76,12 @@ G_BEGIN_DECLS * One slot for the gl thread to render the current frame while another slot is * available for queuing the next audio buffer to render. * - * Note: Increasing the number of slots >2 is not fully supported. - * GstSegments won't be handled correctly currently. See inline code comments. + * Note: Increasing the number of slots >2 is not fully supported since + * it would require handling of PTS offset changes. See inline code comments. * * Valid values: - * 1 - Wait for previous render to complete before scheduling. - * 2 - Render one item and schedule another at the same time. + * 1 : Wait for previous render to complete before scheduling. + * 2 : Render one item and schedule another at the same time. */ #ifndef NUM_RENDER_SLOTS #define NUM_RENDER_SLOTS 2 @@ -89,11 +89,19 @@ G_BEGIN_DECLS /** * Max number of gl frame buffers waiting in a scheduled state to be pushed. - * Capacity should be low. Queuing call will block when capacity is reached and + * The push queue decouples the render loop from buffer push timing, allowing + * the render loop to render frames ahead up to the queue capacity. + * Capacity should be low (1-2) to allow back-pressure from fps increases to + * propagate quickly. The queuing call will block when capacity is reached and * throttle the render loop, frames are never dropped. + * + * 0 : Disable push queuing, block render loop directly until PTS of current + * frame is reached. Disables the push queue API entirely. + * >0 : Allow n buffers waiting in the queue for pushing while render thread + * continues. */ -#ifndef PUSH_QUEUE_MAX_SIZE -#define PUSH_QUEUE_MAX_SIZE 2 +#ifndef PUSH_QUEUE_SIZE +#define PUSH_QUEUE_SIZE 1 #endif /** @@ -246,7 +254,7 @@ typedef struct { /** * Ring buffer to schedule gl buffers for pushing. */ - GstBuffer *push_queue[PUSH_QUEUE_MAX_SIZE]; + GstBuffer *push_queue[PUSH_QUEUE_SIZE]; /** * Mutex for push ring buffer. From d2d675e69aee1b2b0a349363077bead5d1d89fc9 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sun, 12 Oct 2025 19:55:41 -0500 Subject: [PATCH 18/32] fix compile time setting --- src/gstglbaseaudiovisualizer.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index c5194a4..d747c96 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -72,13 +72,20 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); #define DEFAULT_TIMESTAMP_OFFSET 0 /** - * Allow 0.75 * fps frame duration as wait time for frame render queuing. + * Allow 0.75 * fps frame duration as wait time for frame render queuing before + * dropping previous frame. */ #ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N #define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N 3 +#endif + +#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D #define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D 4 #endif +/* + * GST element property default values. + */ #define DEFAULT_MIN_FPS_N 1 #define DEFAULT_MIN_FPS_D 1 #define DEFAULT_PIPELINE_LIVE "auto" From c5fae1f85471317e4306743ec36b0411531b765f Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 13 Oct 2025 00:14:13 -0500 Subject: [PATCH 19/32] add clock jitter average --- src/renderbuffer.c | 100 +++++++++++++++++++++++++++++++++++++++------ src/renderbuffer.h | 16 ++++++++ 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index e85cda4..ab4e735 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -3,6 +3,8 @@ #include "config.h" #endif +#include + #include "renderbuffer.h" GST_DEBUG_CATEGORY_STATIC(renderbuffer_debug); @@ -21,7 +23,7 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; #endif /** - * EMA alpha = 0.25 + * EMA alpha = 0.2 */ #ifndef RB_EMA_ALPHA_N #define RB_EMA_ALPHA_N 1 @@ -33,14 +35,14 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; /** * EMA increase frame duration (slow down fps) in case of detected lag. - * +20% + * +15% */ #ifndef RB_EMA_FRAME_DURATION_INCREASE_N -#define RB_EMA_FRAME_DURATION_INCREASE_N 12 +#define RB_EMA_FRAME_DURATION_INCREASE_N 115 #endif #ifndef RB_EMA_FRAME_DURATION_INCREASE_D -#define RB_EMA_FRAME_DURATION_INCREASE_D 10 +#define RB_EMA_FRAME_DURATION_INCREASE_D 100 #endif /** @@ -69,7 +71,7 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; /** * EMA tolerance for being too fast. - * allow render time as low as 0.9x + * allow render time as low as 0.95x */ #ifndef RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N #define RB_EMA_FRAME_DURATION_TOLERANCE_DOWN_N 95 @@ -85,7 +87,7 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; * a defined max run-time of the scheduling process. */ #ifndef MIN_FREE_SLOT_SCHEDULE_WAIT -#define MIN_FREE_SLOT_SCHEDULE_WAIT (GST_MSECOND * 1) +#define MIN_FREE_SLOT_SCHEDULE_WAIT GST_MSECOND #endif /** @@ -94,7 +96,21 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; * skipped. */ #ifndef MIN_PUSH_SCHEDULE_WAIT -#define MIN_PUSH_SCHEDULE_WAIT (GST_USECOND * 20) +#define MIN_PUSH_SCHEDULE_WAIT (GST_USECOND * 50) +#endif + +/** + * EMA aloha for push schedule clock jitter average. + */ +#ifndef JITTER_EMA_ALPHA +#define JITTER_EMA_ALPHA 0.75 +#endif + +/** + * EMA for push schedule clock jitter outlier threshold. + */ +#ifndef JITTER_EMA_OUTLIER_THRESHOLD +#define JITTER_EMA_OUTLIER_THRESHOLD (5 * GST_MSECOND) #endif /** @@ -251,6 +267,9 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, // init clean up queue state->cleanup_thread = NULL; state->buffer_cleanup_queue = g_async_queue_new(); + + state->avg_jitter = 0.0; + state->avg_jitter_init = FALSE; } void rb_dispose_render_buffer(RBRenderBuffer *state) { @@ -452,7 +471,7 @@ gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, * @param context Current gl context. * @param buf GL buffer to release. */ -static void gl_buffer_cleanup(GstGLContext *context, gpointer buf) { +static void rb_cb_gl_buffer_cleanup(GstGLContext *context, gpointer buf) { (void)context; gst_buffer_unref(GST_BUFFER(buf)); } @@ -477,7 +496,7 @@ static gpointer rb_cleanup_thread_func(gpointer user_data) { if (!item || item == RB_Q_SHUTDOWN_SIGNAL) continue; - gst_gl_context_thread_add(state->gl_context, gl_buffer_cleanup, item); + gst_gl_context_thread_add(state->gl_context, rb_cb_gl_buffer_cleanup, item); } return NULL; } @@ -582,6 +601,44 @@ static void rb_clear_push_queue(RBRenderBuffer *state) { } #endif +/** + * Calculate current clock jitter average. + * + * @param state Current render buffer. + * @param jitter Latest jitter value. + */ +static void rb_calculate_avg_jitter(RBRenderBuffer *state, + GstClockTimeDiff jitter) { + // Ignore outliers + if (ABS(jitter) > JITTER_EMA_OUTLIER_THRESHOLD) + return; + + if (!state->avg_jitter_init) { + state->avg_jitter = (gdouble)jitter; + state->avg_jitter_init = TRUE; + } else { + state->avg_jitter = JITTER_EMA_ALPHA * state->avg_jitter + + (1.0 - JITTER_EMA_ALPHA) * (gdouble)jitter; + } +} + +/** + * Applies jitter correction to the given buffer. + * + * @param state Current render buffer. + * @param outbuf Buffer to apply correction to. + */ +static void rb_jitter_correction(RBRenderBuffer *state, GstBuffer *outbuf) { + GstClockTime correction = + gst_util_uint64_scale_int((guint64)fabs(state->avg_jitter), 1, 1); + + if (state->avg_jitter > 0) { + GST_BUFFER_PTS(outbuf) -= correction; + } else { + GST_BUFFER_PTS(outbuf) += correction; + } +} + /** * Pushes gl buffers for real-time rendering only. * Consume buffers to push and wait until it's PTS time to push. @@ -633,9 +690,13 @@ static gpointer rb_push_thread_func(gpointer user_data) { GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { + GstClockTimeDiff jitter = 0; GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); - gst_clock_id_wait(clock_id, NULL); + gst_clock_id_wait(clock_id, &jitter); gst_clock_id_unref(clock_id); + + // record jitter + rb_calculate_avg_jitter(state, jitter); } gst_object_unref(clock); @@ -646,6 +707,9 @@ static gpointer rb_push_thread_func(gpointer user_data) { g_cond_signal(&state->push_queue_free_cond); g_mutex_unlock(&state->push_queue_mutex); + // apply wait jitter correction ro buffer + rb_jitter_correction(state, outbuf); + // push buffer downstream const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); @@ -711,10 +775,14 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, gst_element_get_base_time(GST_ELEMENT(state->plugin)); const GstClockTime abs_time = pts + base_time; + GstClockTimeDiff jitter = 0; GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); - gst_clock_id_wait(clock_id, NULL); + gst_clock_id_wait(clock_id, &jitter); gst_clock_id_unref(clock_id); + + rb_calculate_avg_jitter(state, jitter); + rb_jitter_correction(state, outbuf); } gst_object_unref(clock); } @@ -937,17 +1005,25 @@ static gpointer rb_render_thread_func(gpointer user_data) { return NULL; } +/** + * Release all buffers currently queued for disposal. + * Needs to be called from GL thread. + * + * @param state Renderbuffer owning cleanup queue to clear. + */ static void rb_clear_cleanup_queue(RBRenderBuffer *state) { g_assert(state != NULL); + // make sure all gl buffers are released gpointer item; while ((item = g_async_queue_try_pop(state->buffer_cleanup_queue)) != NULL) { - gst_gl_context_thread_add(state->gl_context, gl_buffer_cleanup, item); + rb_cb_gl_buffer_cleanup(NULL, item); } } /** * Clears all queues. + * Needs to be called from GL thread. * * @param state Render buffer to clear. */ diff --git a/src/renderbuffer.h b/src/renderbuffer.h index be5b23a..8ef62d4 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -371,6 +371,19 @@ typedef struct { */ gint push_queue_read_idx; + // used only by either render or push thread + // -------------------------------------------------------------- + + /** + * EMA based clock jitter average. + */ + gdouble avg_jitter; + + /** + * Clock jitter initialized. + */ + gboolean avg_jitter_init; + } RBRenderBuffer; /** @@ -483,6 +496,7 @@ static gboolean rb_is_render_too_late(GstElement *element, GstClockTime latency, /** * Clears all queues. + * Needs to be called from GL thread. * * @param state Render buffer to clear. */ @@ -490,6 +504,7 @@ void rb_clear(RBRenderBuffer *state); /** * Start render loop. + * Needs to be called from GL thread. * * @param state Render buffer to use. * @param gl_context GL context to use for rendering. @@ -499,6 +514,7 @@ void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, GstPad *src_pad); /** * Stop render loop. Active threads will be joined before returning. + * Needs to be called from GL thread. * * @param state Render buffer to use. */ From 7293c5fe44b27d05203fe10ead0af929803cc119 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 13 Oct 2025 01:23:10 -0500 Subject: [PATCH 20/32] fix lock congestion --- src/renderbuffer.c | 74 ++++++++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index ab4e735..8db8d5b 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -629,13 +629,15 @@ static void rb_calculate_avg_jitter(RBRenderBuffer *state, * @param outbuf Buffer to apply correction to. */ static void rb_jitter_correction(RBRenderBuffer *state, GstBuffer *outbuf) { - GstClockTime correction = - gst_util_uint64_scale_int((guint64)fabs(state->avg_jitter), 1, 1); - if (state->avg_jitter > 0) { - GST_BUFFER_PTS(outbuf) -= correction; - } else { - GST_BUFFER_PTS(outbuf) += correction; + if (GST_BUFFER_PTS(outbuf) != GST_CLOCK_TIME_NONE) { + GstClockTime correction = llabs((guint64)state->avg_jitter); + + if (state->avg_jitter > 0) { + GST_BUFFER_PTS(outbuf) -= correction; + } else { + GST_BUFFER_PTS(outbuf) += correction; + } } } @@ -678,27 +680,41 @@ static gpointer rb_push_thread_func(gpointer user_data) { GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; // determine when it's time to push - GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - const GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - - const GstClockTime pts = GST_BUFFER_PTS(outbuf); - const GstClockTime abs_time = pts + base_time; - - GstClockTimeDiff remaining_wait = - GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); + const GstClockTime pts = GST_BUFFER_PTS(outbuf); + if (pts != GST_CLOCK_TIME_NONE) { + GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + if (clock) { + const GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); - if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { - GstClockTimeDiff jitter = 0; - GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); - gst_clock_id_wait(clock_id, &jitter); - gst_clock_id_unref(clock_id); + const GstClockTime abs_time = pts + base_time; - // record jitter - rb_calculate_avg_jitter(state, jitter); + GstClockTimeDiff remaining_wait = + GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); + + if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { + g_mutex_unlock(&state->push_queue_mutex); + + GstClockTimeDiff jitter = 0; + GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); + GstClockReturn clock_return = gst_clock_id_wait(clock_id, &jitter); + gst_clock_id_unref(clock_id); + + if (clock_return == GST_CLOCK_OK || clock_return == GST_CLOCK_EARLY) { + // record jitter + rb_calculate_avg_jitter(state, jitter); + } else if (clock_return == GST_CLOCK_UNSCHEDULED) { + g_mutex_lock(&state->push_queue_mutex); + if (state->push_queue[state->push_queue_read_idx] == outbuf) { + state->push_queue[state->push_queue_read_idx] = NULL; + rb_queue_gl_buffer_cleanup(state, outbuf); + } + gst_object_unref(clock); + continue; + } + g_mutex_lock(&state->push_queue_mutex); + } } - gst_object_unref(clock); } @@ -713,7 +729,10 @@ static gpointer rb_push_thread_func(gpointer user_data) { // push buffer downstream const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); - if (ret != GST_FLOW_OK) { + if (ret == GST_FLOW_FLUSHING) { + GST_INFO_OBJECT(state->plugin, + "Pad is flushing and does not accept buffers anymore"); + } else if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } @@ -790,7 +809,10 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, // push buffer downstream directly for offline rendering or if // queuing is disabled ret = gst_pad_push(state->src_pad, outbuf); - if (ret != GST_FLOW_OK) { + if (ret == GST_FLOW_FLUSHING) { + GST_INFO_OBJECT(state->plugin, + "Pad is flushing and does not accept buffers anymore"); + } else if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } #if PUSH_QUEUE_SIZE > 0 From ae13a1e5443561bd82d194861f0f1b4fdfb8eea2 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 13 Oct 2025 01:54:29 -0500 Subject: [PATCH 21/32] fix log init --- src/gstglbaseaudiovisualizer.h | 14 ++++++++++++++ src/register.c | 7 ------- src/renderbuffer.c | 10 ++++++---- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index 4e58a8f..090f833 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -64,9 +64,23 @@ GType gst_gl_base_audio_visualizer_get_type(void); (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_GL_BASE_AUDIO_VISUALIZER, \ GstGLBaseAudioVisualizerClass)) +/** + * Plugin mode of operation type. + */ typedef enum { + /** + * Real-time / live rendering. + */ GST_GL_BASE_AUDIO_VISUALIZER_REALTIME, + + /** + * Faster-than-real-time rendering. + */ GST_GL_BASE_AUDIO_VISUALIZER_OFFLINE, + + /** + * Auto-detect if pipeline is live. + */ GST_GL_BASE_AUDIO_VISUALIZER_AUTO } GstGLBaseAudioVisualizerMode; diff --git a/src/register.c b/src/register.c index 3065cd6..afe3b5d 100644 --- a/src/register.c +++ b/src/register.c @@ -12,17 +12,10 @@ * This unit registers all gst elements from this plugin library to make them * available to GStreamer. */ - -GST_DEBUG_CATEGORY(gst_projectm_debug); -#define GST_CAT_DEFAULT gst_projectm_debug - static gboolean plugin_init(GstPlugin *plugin) { gst_projectm_base_init_once(); - GST_DEBUG_CATEGORY_INIT(gst_projectm_debug, "projectm", 0, - "projectM visualizer plugin"); - // register main plugin projectM element gboolean p1 = gst_element_register(plugin, "projectm", GST_RANK_NONE, GST_TYPE_PROJECTM); diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 8db8d5b..8c5c608 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -194,10 +194,10 @@ static void rb_handle_adaptive_fps_ema(RBRenderBuffer *state, } } -static void rb_queue_gl_buffer_cleanup(RBRenderBuffer *state, GstBuffer *out) { +static void rb_queue_gl_buffer_cleanup(RBRenderBuffer *state, GstBuffer *buf) { g_assert(state != NULL); - g_assert(out != NULL); - g_async_queue_push(state->buffer_cleanup_queue, out); + g_assert(buf != NULL); + g_async_queue_push(state->buffer_cleanup_queue, buf); } void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, @@ -633,7 +633,7 @@ static void rb_jitter_correction(RBRenderBuffer *state, GstBuffer *outbuf) { if (GST_BUFFER_PTS(outbuf) != GST_CLOCK_TIME_NONE) { GstClockTime correction = llabs((guint64)state->avg_jitter); - if (state->avg_jitter > 0) { + if (state->avg_jitter > 0.0) { GST_BUFFER_PTS(outbuf) -= correction; } else { GST_BUFFER_PTS(outbuf) += correction; @@ -693,6 +693,7 @@ static gpointer rb_push_thread_func(gpointer user_data) { GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { + // we need to wait, unlock first g_mutex_unlock(&state->push_queue_mutex); GstClockTimeDiff jitter = 0; @@ -704,6 +705,7 @@ static gpointer rb_push_thread_func(gpointer user_data) { // record jitter rb_calculate_avg_jitter(state, jitter); } else if (clock_return == GST_CLOCK_UNSCHEDULED) { + // drop buffer if clock is not running g_mutex_lock(&state->push_queue_mutex); if (state->push_queue[state->push_queue_read_idx] == outbuf) { state->push_queue[state->push_queue_read_idx] = NULL; From 6f581c513ad3c7ea5d04769b682eb4cf1d203eb8 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 13 Oct 2025 12:17:46 -0500 Subject: [PATCH 22/32] refactor renderbuffer into more manageable units --- CMakeLists.txt | 14 +- src/bufferdisposal.c | 111 +++++++++ src/bufferdisposal.h | 88 ++++++++ src/gstglbaseaudiovisualizer.c | 12 +- src/gstpmaudiovisualizer.c | 23 +- src/pushbuffer.c | 271 ++++++++++++++++++++++ src/pushbuffer.h | 188 ++++++++++++++++ src/renderbuffer.c | 395 +++------------------------------ src/renderbuffer.h | 117 ++-------- 9 files changed, 733 insertions(+), 486 deletions(-) create mode 100644 src/bufferdisposal.c create mode 100644 src/bufferdisposal.h create mode 100644 src/pushbuffer.c create mode 100644 src/pushbuffer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7303333..06f8850 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,19 +15,23 @@ find_package(GStreamer REQUIRED COMPONENTS gstreamer-audio gstreamer-gl gstreame find_package(GLIB2 REQUIRED) add_library(gstprojectm SHARED - src/gstprojectmcaps.h - src/gstprojectmcaps.c + src/bufferdisposal.h + src/bufferdisposal.c src/debug.h src/debug.c - src/gstprojectmconfig.h - src/gstprojectm.h - src/gstprojectm.c src/gstglbaseaudiovisualizer.h src/gstglbaseaudiovisualizer.c src/gstpmaudiovisualizer.h src/gstpmaudiovisualizer.c + src/gstprojectm.h + src/gstprojectm.c src/gstprojectmbase.h src/gstprojectmbase.c + src/gstprojectmcaps.h + src/gstprojectmcaps.c + src/gstprojectmconfig.h + src/pushbuffer.h + src/pushbuffer.c src/register.c src/renderbuffer.h src/renderbuffer.c diff --git a/src/bufferdisposal.c b/src/bufferdisposal.c new file mode 100644 index 0000000..879da3a --- /dev/null +++ b/src/bufferdisposal.c @@ -0,0 +1,111 @@ + +#include "bufferdisposal.h" + +GST_DEBUG_CATEGORY_STATIC(buffercleanup_debug); +#define GST_CAT_DEFAULT buffercleanup_debug + +/** + * Queue shutdown signal token. + */ +static gpointer BC_Q_SHUTDOWN_SIGNAL = &BC_Q_SHUTDOWN_SIGNAL; + +void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf) { + g_assert(state != NULL); + g_assert(buf != NULL); + g_async_queue_push(state->disposal_queue, buf); +} + +/** + * Callback for scheduling gl buffer release with gl thread. + * Needs to be called from the GL thread. + * + * @param context Current gl context. + * @param buf GL buffer to release. + */ +void bd_gl_buffer_dispose_gl(GstGLContext *context, gpointer buf) { + (void)context; + gst_buffer_unref(GST_BUFFER(buf)); +} + +/** + * Used to dispose of dropped gl buffers that are not making it to the src pad. + * Consume buffers to clean-up and dispatch release through gl thread. + * + * @param user_data Queue state to use. + * @return NULL + */ +static gpointer bd_dispose_thread_func(gpointer user_data) { + + BDBufferDisposal *state = (BDBufferDisposal *)user_data; + g_assert(state != NULL); + + // consume gl buffers to dispatch to gl thread for cleanup + while (g_atomic_int_get(&state->running)) { + + gpointer item = g_async_queue_pop(state->disposal_queue); + + if (!item || item == BC_Q_SHUTDOWN_SIGNAL) + continue; + + gst_gl_context_thread_add(state->gl_context, bd_gl_buffer_dispose_gl, item); + } + return NULL; +} + +/** + * Dispose of all buffered currently queued. + * Needs to be called from the GL thread. + * + * @param user_data Queue state to use. + * @return NULL + */ +void bd_clear_queue_gl(GstGLContext* context, gpointer user_data) { + BDBufferDisposal *state = (BDBufferDisposal *)user_data; + g_assert(state != NULL); + + // make sure all gl buffers are released + gpointer item; + while ((item = g_async_queue_try_pop(state->disposal_queue)) != NULL) { + bd_gl_buffer_dispose_gl(context, item); + } +} + +void bd_clear_disposal_queue(BDBufferDisposal *state) { + gst_gl_context_thread_add(state->gl_context, bd_clear_queue_gl, state); +} + +void bd_init_buffer_disposal(BDBufferDisposal *state, + GstGLContext *gl_context) { + + GST_DEBUG_CATEGORY_INIT(buffercleanup_debug, "buffercleanup", 0, + "projectM visualizer plugin buffer cleanup"); + + // init clean up queue + state->disposal_thread = NULL; + state->gl_context = gl_context; + g_atomic_int_set(&state->running, FALSE); + state->disposal_queue = g_async_queue_new(); +} + +void bd_dispose_buffer_disposal(BDBufferDisposal *state) { + g_async_queue_unref(state->disposal_queue); + state->disposal_queue = NULL; + state->gl_context = NULL; +} + +void bd_start_buffer_disposal(BDBufferDisposal *state) { + + g_atomic_int_set(&state->running, TRUE); + + state->disposal_thread = + g_thread_new("rb-cleanup-thread", bd_dispose_thread_func, state); +} + +void bd_stop_buffer_disposal(BDBufferDisposal *state) { + // signal and wait for cleanup thread to exit + g_atomic_int_set(&state->running, FALSE); + + g_async_queue_push(state->disposal_queue, BC_Q_SHUTDOWN_SIGNAL); + g_thread_join(state->disposal_thread); + state->disposal_thread = NULL; +} \ No newline at end of file diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h new file mode 100644 index 0000000..b330341 --- /dev/null +++ b/src/bufferdisposal.h @@ -0,0 +1,88 @@ +/* + * An async queue to dispose of dropped GL buffers. It is being consumed by + * a dedicated thread (disposal thread) to dispatch GL buffer cleanup to the GL + * thread. + */ + +#ifndef __BUFFERDISPOSAL_H__ +#define __BUFFERDISPOSAL_H__ + +#include +#include + +typedef struct { + + // not re-assigned during render thread lifetime + // -------------------------------------------------------------- + + /** + * Current gl context. No ownership. + */ + GstGLContext *gl_context; + + /** + * Thread running the gl buffer clean-up loop, + * used to release dropped buffer from the gl thread. + */ + GThread *disposal_thread; + + /** + * Queue to dispose of dropped gl buffers. + */ + GAsyncQueue *disposal_queue; + + // concurrent access, g_atomic + // -------------------------------------------------------------- + + /** + * TRUE if rendering is currently running. + */ + gboolean running; + +} BDBufferDisposal; + +/** + * Release given buffer from the GL thread. + * + * @param state State to use. + * @param buf Buffer to dispose. + */ +void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf); + +/** + * Dispose of all buffers currently queued for disposal. + * + * @param state Renderbuffer owning cleanup queue to clear. + */ +void bd_clear_disposal_queue(BDBufferDisposal *state); + +/** + * Init queue state. + * + * @param state Queue state to init. + * @param gl_context GL context to use. + */ +void bd_init_buffer_disposal(BDBufferDisposal *state, GstGLContext *gl_context); + +/** + * Release all resources used by this queue. + * + * @param state Queue state to dispose. + */ +void bd_dispose_buffer_disposal(BDBufferDisposal *state); + +/** + * Start worker thread. + * + * @param state Queue state to use. + */ +void bd_start_buffer_disposal(BDBufferDisposal *state); + +/** + * Stop worker thread. + * + * @param state Queue state to use. + */ +void bd_stop_buffer_disposal(BDBufferDisposal *state); + +#endif \ No newline at end of file diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index d747c96..38d07ea 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -483,17 +483,17 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, } // render loop QoS is disabled for offline rendering - rb_init_render_buffer(&glav->priv->render_buffer, GST_OBJECT(glav), - gst_gl_base_audio_visualizer_fill_gl, - adjust_fps_callback, max_frame_duration, - caps_frame_duration, glav->priv->is_realtime, - glav->priv->is_realtime); + rb_init_render_buffer( + &glav->priv->render_buffer, GST_OBJECT(glav), glav->context, pmav->srcpad, + gst_gl_base_audio_visualizer_fill_gl, adjust_fps_callback, + max_frame_duration, caps_frame_duration, glav->priv->is_realtime, + glav->priv->is_realtime); // cascade gl start to implementor glav->priv->gl_started = glav_class->gl_start(glav); // get gl rendering going - rb_start(&glav->priv->render_buffer, glav->context, pmav->srcpad); + rb_start(&glav->priv->render_buffer); } static void diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 35928c8..ddaeaa6 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -438,6 +438,16 @@ static gboolean gst_pm_audio_visualizer_do_setup(GstPMAudioVisualizer *scope) { return TRUE; } +static void check_ready_unlocked(GstPMAudioVisualizer *scope) { + if (scope->priv->src_ready && scope->priv->sink_ready) { + g_mutex_unlock(&scope->priv->config_lock); + gst_pm_audio_visualizer_do_setup(scope); + g_mutex_lock(&scope->priv->config_lock); + } else { + scope->priv->ready = FALSE; + } +} + static gboolean gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, GstCaps *caps) { @@ -459,12 +469,9 @@ gst_pm_audio_visualizer_sink_setcaps(GstPMAudioVisualizer *scope, g_mutex_lock(&scope->priv->config_lock); scope->priv->sink_ready = TRUE; + check_ready_unlocked(scope); g_mutex_unlock(&scope->priv->config_lock); - if (scope->priv->src_ready) { - gst_pm_audio_visualizer_do_setup(scope); - } - return TRUE; /* Errors */ @@ -504,12 +511,8 @@ static gboolean gst_pm_audio_visualizer_src_setcaps(GstPMAudioVisualizer *scope, g_mutex_lock(&scope->priv->config_lock); scope->priv->src_ready = TRUE; + check_ready_unlocked(scope); g_mutex_unlock(&scope->priv->config_lock); - if (scope->priv->sink_ready) { - if (!gst_pm_audio_visualizer_do_setup(scope)) { - goto setup_failed; - } - } return res; @@ -1215,7 +1218,7 @@ void gst_pm_audio_visualizer_adjust_fps(GstPMAudioVisualizer *scope, gchar *message = g_strdup_printf("Adjusting framerate, max fps: %f, using " "frame-duration: %" GST_TIME_FORMAT ", spf: %u", - (gdouble)frame_duration / GST_SECOND, + (gdouble)GST_SECOND / (gdouble)frame_duration, GST_TIME_ARGS(set_duration), set_req_spf); g_idle_add(log_fps_change, message); diff --git a/src/pushbuffer.c b/src/pushbuffer.c new file mode 100644 index 0000000..74dc318 --- /dev/null +++ b/src/pushbuffer.c @@ -0,0 +1,271 @@ + +#include "pushbuffer.h" + +#include "bufferdisposal.h" + +GST_DEBUG_CATEGORY_STATIC(pushbuffer_debug); +#define GST_CAT_DEFAULT pushbuffer_debug + +/** + * EMA aloha for push schedule clock jitter average. + */ +#ifndef JITTER_EMA_ALPHA +#define JITTER_EMA_ALPHA 0.75 +#endif + +/** + * EMA for push schedule clock jitter outlier threshold. + */ +#ifndef JITTER_EMA_OUTLIER_THRESHOLD +#define JITTER_EMA_OUTLIER_THRESHOLD (5 * GST_MSECOND) +#endif + +/** + * Tolerance / minimal wait time for scheduling a timed wait before pushing a + * buffer. If the calculated wait time is less than this value, the wait will be + * skipped. + */ +#ifndef MIN_PUSH_SCHEDULE_WAIT +#define MIN_PUSH_SCHEDULE_WAIT (GST_USECOND * 50) +#endif + +gboolean pb_queue_buffer(PBPushBuffer *state, GstBuffer *buffer) { + g_assert(state != NULL); + g_assert(buffer != NULL); + + g_mutex_lock(&state->push_queue_mutex); + + // write to the next position in the ring + state->push_queue_write_idx = + (state->push_queue_write_idx + 1) % PUSH_QUEUE_SIZE; + + gboolean result = TRUE; + // wait until next position is free + while (state->push_queue[state->push_queue_write_idx] != NULL) { + g_cond_wait(&state->push_queue_free_cond, &state->push_queue_mutex); + if (!g_atomic_int_get(&state->running)) { + result = FALSE; + break; + } + } + + if (result) { + // take the spot and signal that queue has changed + state->push_queue[state->push_queue_write_idx] = buffer; + g_cond_signal(&state->push_queue_cond); + } + + g_mutex_unlock(&state->push_queue_mutex); + + return result; +} + +void pb_calculate_avg_jitter(PBPushBuffer *state, + const GstClockTimeDiff jitter) { + // ignore outliers + if (ABS(jitter) > JITTER_EMA_OUTLIER_THRESHOLD) + return; + + if (!state->avg_jitter_init) { + state->avg_jitter = (gdouble)jitter; + state->avg_jitter_init = TRUE; + } else { + state->avg_jitter = JITTER_EMA_ALPHA * state->avg_jitter + + (1.0 - JITTER_EMA_ALPHA) * (gdouble)jitter; + } +} + +static void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf) { + + if (GST_BUFFER_PTS(outbuf) != GST_CLOCK_TIME_NONE) { + GstClockTime correction = llabs((guint64)state->avg_jitter); + + if (state->avg_jitter > 0.0) { + GST_BUFFER_PTS(outbuf) -= correction; + } else { + GST_BUFFER_PTS(outbuf) += correction; + } + } +} + +/** + * Consume buffers to push and wait until it's PTS time to push. + * Used for real-time rendering only. + * + * + * @param user_data Render buffer to use. + * @return NULL + */ +static gpointer rb_push_thread_func(gpointer user_data) { + + PBPushBuffer *state = (PBPushBuffer *)user_data; + g_assert(state != NULL); + + g_mutex_lock(&state->push_queue_mutex); + + while (g_atomic_int_get(&state->running)) { + + state->push_queue_read_idx = + (state->push_queue_read_idx + 1) % PUSH_QUEUE_SIZE; + + // consume gl buffer to push + gboolean stop = FALSE; + while (state->push_queue[state->push_queue_read_idx] == NULL) { + // no buffer to push, wait for one + g_cond_wait(&state->push_queue_cond, &state->push_queue_mutex); + if (!g_atomic_int_get(&state->running)) { + stop = TRUE; + break; + } + } + + if (stop) { + break; + } + + // found a buffer to push + GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; + + // determine when it's time to push + const GstClockTime pts = GST_BUFFER_PTS(outbuf); + if (pts != GST_CLOCK_TIME_NONE) { + GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + if (clock) { + const GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + + const GstClockTime abs_time = pts + base_time; + + const GstClockTimeDiff remaining_wait = + GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); + + if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { + // we need to wait, unlock first + g_mutex_unlock(&state->push_queue_mutex); + + GstClockTimeDiff jitter = 0; + const GstClockID clock_id = + gst_clock_new_single_shot_id(clock, abs_time); + const GstClockReturn clock_return = + gst_clock_id_wait(clock_id, &jitter); + gst_clock_id_unref(clock_id); + + if (clock_return == GST_CLOCK_OK || clock_return == GST_CLOCK_EARLY) { + // record jitter + pb_calculate_avg_jitter(state, jitter); + } else if (clock_return == GST_CLOCK_UNSCHEDULED) { + // drop buffer if clock is not running + g_mutex_lock(&state->push_queue_mutex); + if (state->push_queue[state->push_queue_read_idx] == outbuf) { + state->push_queue[state->push_queue_read_idx] = NULL; + bd_queue_gl_buffer_disposal(state->buffer_disposal, outbuf); + } + gst_object_unref(clock); + continue; + } + g_mutex_lock(&state->push_queue_mutex); + } + } + gst_object_unref(clock); + } + + // now we own the buffer to push + state->push_queue[state->push_queue_read_idx] = NULL; + g_cond_signal(&state->push_queue_free_cond); + g_mutex_unlock(&state->push_queue_mutex); + + // apply wait jitter correction ro buffer + pb_jitter_correction(state, outbuf); + + // push buffer downstream + const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); + + if (ret == GST_FLOW_FLUSHING) { + GST_INFO_OBJECT(state->plugin, + "Pad is flushing and does not accept buffers anymore"); + } else if (ret != GST_FLOW_OK) { + GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); + } + + g_mutex_lock(&state->push_queue_mutex); + } + + g_mutex_unlock(&state->push_queue_mutex); + + return NULL; +} + +void pb_init_push_buffer(PBPushBuffer *state, BDBufferDisposal *buffer_cleanup, + GstObject *plugin, GstPad *src_pad) { + + GST_DEBUG_CATEGORY_INIT(pushbuffer_debug, "pushbuffer", 0, + "projectM visualizer plugin push buffer"); + + state->buffer_disposal = buffer_cleanup; + state->plugin = plugin; + state->src_pad = src_pad; + + // init push queue + state->push_thread = NULL; + state->push_queue_read_idx = -1; + state->push_queue_write_idx = -1; + g_mutex_init(&state->push_queue_mutex); + g_cond_init(&state->push_queue_cond); + g_cond_init(&state->push_queue_free_cond); + for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { + state->push_queue[i] = NULL; + } + + state->avg_jitter = 0.0; + state->avg_jitter_init = FALSE; +} + +void pb_dispose_push_buffer(PBPushBuffer *state) { + + g_cond_clear(&state->push_queue_cond); + g_cond_clear(&state->push_queue_free_cond); + g_mutex_clear(&state->push_queue_mutex); + + state->buffer_disposal = NULL; + state->plugin = NULL; + state->src_pad = NULL; +} + +void pb_clear_queue(PBPushBuffer *state) { + g_assert(state != NULL); + g_mutex_lock(&state->push_queue_mutex); + + // release buffers that are still queued before cleanup thread shuts down + for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { + if (state->push_queue[i] != NULL) { + bd_queue_gl_buffer_disposal(state->buffer_disposal, state->push_queue[i]); + state->push_queue[i] = NULL; + } + } + state->push_queue_read_idx = -1; + state->push_queue_write_idx = -1; + state->avg_jitter = 0.0; + state->avg_jitter_init = FALSE; + + g_mutex_unlock(&state->push_queue_mutex); +} + +void pb_start_push_buffer(PBPushBuffer *state) { + g_atomic_int_set(&state->running, TRUE); + + state->push_thread = + g_thread_new("rb-push-thread", rb_push_thread_func, state); +} + +void pb_stop_push_buffer(PBPushBuffer *state) { + g_atomic_int_set(&state->running, FALSE); + + // signal wake up push thread to signal loop exit + g_mutex_lock(&state->push_queue_mutex); + g_cond_broadcast(&state->push_queue_cond); + g_cond_broadcast(&state->push_queue_free_cond); + g_mutex_unlock(&state->push_queue_mutex); + // wait for push thread to exit + g_thread_join(state->push_thread); + state->push_thread = NULL; +} diff --git a/src/pushbuffer.h b/src/pushbuffer.h new file mode 100644 index 0000000..a7245ec --- /dev/null +++ b/src/pushbuffer.h @@ -0,0 +1,188 @@ +/* + * A ring buffer based queue to schedule GL buffers to be pushed + * downstream at presentation time (PTS). It is being consumed by a dedicated + * thread (push thread) used for processing. The queuing call will block when + * capacity is reached and throttle the render loop by letting it wait for a + * free slot. Frames are never dropped. + */ + +#ifndef __PUSHBUFFER_H__ +#define __PUSHBUFFER_H__ + +#include + +#include "bufferdisposal.h" + +/** + * Max number of gl frame buffers waiting in a scheduled state to be pushed. + * The push queue decouples the render loop from buffer push timing, allowing + * the render loop to render frames ahead up to the queue capacity. + * Capacity should be low (1-2) to allow back-pressure from fps increases to + * propagate quickly. + * + * 0 : Disable push queuing, block render loop directly until PTS of current + * frame is reached. Disables the push queue API entirely. + * >0 : Allow n buffers waiting in the queue for pushing while render thread + * continues. + */ +#ifndef PUSH_QUEUE_SIZE +#define PUSH_QUEUE_SIZE 1 +#endif + +/** + * All render buffer data. + */ +typedef struct { + + // not re-assigned during render thread lifetime + // -------------------------------------------------------------- + + /** + * projectM plugin. No ownership. + */ + BDBufferDisposal *buffer_disposal; + + /** + * projectM plugin. No ownership. + */ + GstObject *plugin; + + /** + * projectM plugin source pad. No ownership. + */ + GstPad *src_pad; + + /** + * Thread for pushing gl buffers downstream. + * Used for real-time, pushing needs to be scheduled to be synchronized with + * the pipeline clock. + */ + GThread *push_thread; + + /** + * Ring buffer to schedule gl buffers for pushing. + */ + GstBuffer *push_queue[PUSH_QUEUE_SIZE]; + + /** + * Mutex for push ring buffer. + */ + GMutex push_queue_mutex; + + /** + * Condition signaled when a buffer has been queued. + */ + GCond push_queue_cond; + + /** + * Condition signaled when a buffer has been pushed + * and a slot if free. + */ + GCond push_queue_free_cond; + + // concurrent access, g_atomic + // -------------------------------------------------------------- + + /** + * TRUE if rendering is currently running. + */ + gboolean running; + + // concurrent access, protected by push_queue_mutex + // -------------------------------------------------------------- + + /** + * Push ring buffer write position. + */ + gint push_queue_write_idx; + + /** + * Push ring buffer read position. + */ + gint push_queue_read_idx; + + // used only by either render or push thread + // -------------------------------------------------------------- + + /** + * EMA based clock jitter average. + */ + gdouble avg_jitter; + + /** + * Clock jitter initialized. + */ + gboolean avg_jitter_init; + +} PBPushBuffer; + +/** + * Schedule a rendered gl buffer for pushing downstream. + * The buffer will not be pushed until it's PTS time is reached. + * This call will block until the buffer can be schduled or + * the render buffer is stopped. + * + * @param state The render buffer to use. + * @param buffer The gl buffer to push. Takes ownership of the buffer. + * + * @return TRUE if the buffer was scheduled successfully, FALSE in case the + * buffer was stopped. + */ +gboolean pb_queue_buffer(PBPushBuffer *state, GstBuffer *buffer); + +/** + * Removes and disposes all queued buffers and resets queue state. + * + * @param state State to clear. + */ +void pb_clear_queue(PBPushBuffer *state); + +/** + * Applies jitter correction to the given buffer. + * + * @param state Current render buffer. + * @param outbuf Buffer to apply correction to. + */ +static void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf); + +/** + * Calculate current clock jitter average. + * + * @param state Current render buffer. + * @param jitter Latest jitter value. + */ +void pb_calculate_avg_jitter(PBPushBuffer *state, GstClockTimeDiff jitter); + +/** + * Init this push buffer. + * + * @param state Push buffer to use. + * @param buffer_cleanup Buffer disposal to use. + * @param plugin Context gst plugin element. + * @param src_pad Source pad to push buffers to. + */ +void pb_init_push_buffer(PBPushBuffer *state, BDBufferDisposal *buffer_cleanup, + GstObject *plugin, GstPad *src_pad); + +/** + * Release all respources for this push buffer. + * + * @param state Push buffer to use. + */ +void pb_dispose_push_buffer(PBPushBuffer *state); + +/** + * Start push buffer worker thread. + * + * @param state Push buffer to use. + */ +void pb_start_push_buffer(PBPushBuffer *state); + +/** + * Stop push buffer worker thread. + * + * @param state Push buffer to use. + */ +void pb_stop_push_buffer(PBPushBuffer *state); + +#endif \ No newline at end of file diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 8c5c608..41f8119 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -10,11 +10,6 @@ GST_DEBUG_CATEGORY_STATIC(renderbuffer_debug); #define GST_CAT_DEFAULT renderbuffer_debug -/** - * Queue shutdown signal token. - */ -static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; - /** * Number of frames inspected by EMA. */ @@ -90,29 +85,6 @@ static gpointer RB_Q_SHUTDOWN_SIGNAL = &RB_Q_SHUTDOWN_SIGNAL; #define MIN_FREE_SLOT_SCHEDULE_WAIT GST_MSECOND #endif -/** - * Tolerance / minimal wait time for scheduling a timed wait before pushing a - * buffer. If the calculated wait time is less than this value, the wait will be - * skipped. - */ -#ifndef MIN_PUSH_SCHEDULE_WAIT -#define MIN_PUSH_SCHEDULE_WAIT (GST_USECOND * 50) -#endif - -/** - * EMA aloha for push schedule clock jitter average. - */ -#ifndef JITTER_EMA_ALPHA -#define JITTER_EMA_ALPHA 0.75 -#endif - -/** - * EMA for push schedule clock jitter outlier threshold. - */ -#ifndef JITTER_EMA_OUTLIER_THRESHOLD -#define JITTER_EMA_OUTLIER_THRESHOLD (5 * GST_MSECOND) -#endif - /** * Exponential Moving Average (EMA)-based adaptive frame duration (fps) * adjustment. Determines desired frame duration change based on the @@ -194,13 +166,8 @@ static void rb_handle_adaptive_fps_ema(RBRenderBuffer *state, } } -static void rb_queue_gl_buffer_cleanup(RBRenderBuffer *state, GstBuffer *buf) { - g_assert(state != NULL); - g_assert(buf != NULL); - g_async_queue_push(state->buffer_cleanup_queue, buf); -} - void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, + GstGLContext *gl_context, GstPad *src_pad, const GstGLContextThreadFunc gl_fill_func, const RBAdjustFpsFunc adjust_fps_func, const GstClockTime max_frame_duration, @@ -216,10 +183,8 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, // context config without ownership state->plugin = plugin; state->adjust_fps_func = adjust_fps_func; - - // we'll get these later - state->gl_context = NULL; - state->src_pad = NULL; + state->gl_context = gl_context; + state->src_pad = src_pad; // never changed after init state->qos_enabled = is_qos_enabled; @@ -253,37 +218,21 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, state->ema_frame_counter = 0; state->ema_smoothed_render_time = 0; - // init push queue - state->push_thread = NULL; - state->push_queue_read_idx = -1; - state->push_queue_write_idx = -1; - g_mutex_init(&state->push_queue_mutex); - g_cond_init(&state->push_queue_cond); - g_cond_init(&state->push_queue_free_cond); - for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { - state->push_queue[i] = NULL; - } - - // init clean up queue - state->cleanup_thread = NULL; - state->buffer_cleanup_queue = g_async_queue_new(); - - state->avg_jitter = 0.0; - state->avg_jitter_init = FALSE; + bd_init_buffer_disposal(&state->buffer_disposal, gl_context); + pb_init_push_buffer(&state->push_buffer, &state->buffer_disposal, plugin, + src_pad); } void rb_dispose_render_buffer(RBRenderBuffer *state) { g_assert(state != NULL); - g_async_queue_unref(state->buffer_cleanup_queue); - g_cond_clear(&state->slot_available_cond); g_cond_clear(&state->render_queued_cond); - g_cond_clear(&state->push_queue_cond); - g_cond_clear(&state->push_queue_free_cond); g_mutex_clear(&state->slot_lock); - g_mutex_clear(&state->push_queue_mutex); + + pb_dispose_push_buffer(&state->push_buffer); + bd_dispose_buffer_disposal(&state->buffer_disposal); } RBQueueResult rb_queue_render_task(RBQueueArgs *args) { @@ -361,7 +310,7 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { } if (slot->out_buf != NULL) { - rb_queue_gl_buffer_cleanup(state, slot->out_buf); + bd_queue_gl_buffer_disposal(&state->buffer_disposal, slot->out_buf); slot->out_buf = NULL; } @@ -465,42 +414,6 @@ gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, return FALSE; } -/** - * Callback for scheduling gl buffer release with gl thread. - * - * @param context Current gl context. - * @param buf GL buffer to release. - */ -static void rb_cb_gl_buffer_cleanup(GstGLContext *context, gpointer buf) { - (void)context; - gst_buffer_unref(GST_BUFFER(buf)); -} - -/** - * Used to dispose of dropped gl buffers only. - * Consume buffers to clean-up and dispatch release through gl thread. - * - * @param user_data Render buffer to use. - * @return NULL - */ -static gpointer rb_cleanup_thread_func(gpointer user_data) { - - RBRenderBuffer *state = (RBRenderBuffer *)user_data; - g_assert(state != NULL); - - // consume gl buffers to dispatch to gl thread for cleanup - while (g_atomic_int_get(&state->running)) { - - gpointer item = g_async_queue_pop(state->buffer_cleanup_queue); - - if (!item || item == RB_Q_SHUTDOWN_SIGNAL) - continue; - - gst_gl_context_thread_add(state->gl_context, rb_cb_gl_buffer_cleanup, item); - } - return NULL; -} - /** * Render one frame for the given slot. * @@ -533,220 +446,6 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { return render_time; } -/** - * Schedule a rendered gl buffer for pushing downstream. - * The buffer will not be pushed until it's PTS time is reached. - * This call will block until the buffer can be schduled or - * the render buffer is stopped. - * - * @param state The render buffer to use. - * @param buffer The gl buffer to push. Takes ownership of the buffer. - * - * @return TRUE if the buffer was scheduled successfully, FALSE in case the - * buffer was stopped. - */ -#if PUSH_QUEUE_SIZE > 0 -static gboolean rb_queue_push_buffer(RBRenderBuffer *state, GstBuffer *buffer) { - g_assert(state != NULL); - g_assert(buffer != NULL); - - g_mutex_lock(&state->push_queue_mutex); - - // write to the next position in the ring - state->push_queue_write_idx = - (state->push_queue_write_idx + 1) % PUSH_QUEUE_SIZE; - - gboolean result = TRUE; - while (state->push_queue[state->push_queue_write_idx] != NULL) { - g_cond_wait(&state->push_queue_free_cond, &state->push_queue_mutex); - if (!g_atomic_int_get(&state->running)) { - result = FALSE; - break; - } - } - - if (result) { - // take the spot and signal that queue has changed - state->push_queue[state->push_queue_write_idx] = buffer; - g_cond_signal(&state->push_queue_cond); - } - - g_mutex_unlock(&state->push_queue_mutex); - - return result; -} -#endif - -/** - * Removes and disposes all queued buffers and resets queue state. - * - * @param state State to clear. - */ -#if PUSH_QUEUE_SIZE > 0 -static void rb_clear_push_queue(RBRenderBuffer *state) { - g_assert(state != NULL); - g_mutex_lock(&state->push_queue_mutex); - - // release buffers that are still queued before cleanup thread shuts down - for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { - if (state->push_queue[i] != NULL) { - rb_queue_gl_buffer_cleanup(state, state->push_queue[i]); - state->push_queue[i] = NULL; - } - } - state->push_queue_read_idx = -1; - state->push_queue_write_idx = -1; - - g_mutex_unlock(&state->push_queue_mutex); -} -#endif - -/** - * Calculate current clock jitter average. - * - * @param state Current render buffer. - * @param jitter Latest jitter value. - */ -static void rb_calculate_avg_jitter(RBRenderBuffer *state, - GstClockTimeDiff jitter) { - // Ignore outliers - if (ABS(jitter) > JITTER_EMA_OUTLIER_THRESHOLD) - return; - - if (!state->avg_jitter_init) { - state->avg_jitter = (gdouble)jitter; - state->avg_jitter_init = TRUE; - } else { - state->avg_jitter = JITTER_EMA_ALPHA * state->avg_jitter + - (1.0 - JITTER_EMA_ALPHA) * (gdouble)jitter; - } -} - -/** - * Applies jitter correction to the given buffer. - * - * @param state Current render buffer. - * @param outbuf Buffer to apply correction to. - */ -static void rb_jitter_correction(RBRenderBuffer *state, GstBuffer *outbuf) { - - if (GST_BUFFER_PTS(outbuf) != GST_CLOCK_TIME_NONE) { - GstClockTime correction = llabs((guint64)state->avg_jitter); - - if (state->avg_jitter > 0.0) { - GST_BUFFER_PTS(outbuf) -= correction; - } else { - GST_BUFFER_PTS(outbuf) += correction; - } - } -} - -/** - * Pushes gl buffers for real-time rendering only. - * Consume buffers to push and wait until it's PTS time to push. - * - * @param user_data Render buffer to use. - * @return NULL - */ -#if PUSH_QUEUE_SIZE > 0 -static gpointer rb_push_thread_func(gpointer user_data) { - - RBRenderBuffer *state = (RBRenderBuffer *)user_data; - g_assert(state != NULL); - - g_mutex_lock(&state->push_queue_mutex); - - while (g_atomic_int_get(&state->running)) { - - state->push_queue_read_idx = - (state->push_queue_read_idx + 1) % PUSH_QUEUE_SIZE; - - // consume gl buffer to push - gboolean stop = FALSE; - while (state->push_queue[state->push_queue_read_idx] == NULL) { - // no buffer to push, wait for one - g_cond_wait(&state->push_queue_cond, &state->push_queue_mutex); - if (!g_atomic_int_get(&state->running)) { - stop = TRUE; - break; - } - } - - if (stop) { - break; - } - - // found a buffer to push - GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; - - // determine when it's time to push - const GstClockTime pts = GST_BUFFER_PTS(outbuf); - if (pts != GST_CLOCK_TIME_NONE) { - GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - const GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - - const GstClockTime abs_time = pts + base_time; - - GstClockTimeDiff remaining_wait = - GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); - - if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { - // we need to wait, unlock first - g_mutex_unlock(&state->push_queue_mutex); - - GstClockTimeDiff jitter = 0; - GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); - GstClockReturn clock_return = gst_clock_id_wait(clock_id, &jitter); - gst_clock_id_unref(clock_id); - - if (clock_return == GST_CLOCK_OK || clock_return == GST_CLOCK_EARLY) { - // record jitter - rb_calculate_avg_jitter(state, jitter); - } else if (clock_return == GST_CLOCK_UNSCHEDULED) { - // drop buffer if clock is not running - g_mutex_lock(&state->push_queue_mutex); - if (state->push_queue[state->push_queue_read_idx] == outbuf) { - state->push_queue[state->push_queue_read_idx] = NULL; - rb_queue_gl_buffer_cleanup(state, outbuf); - } - gst_object_unref(clock); - continue; - } - g_mutex_lock(&state->push_queue_mutex); - } - } - gst_object_unref(clock); - } - - // now we own the buffer to push - state->push_queue[state->push_queue_read_idx] = NULL; - g_cond_signal(&state->push_queue_free_cond); - g_mutex_unlock(&state->push_queue_mutex); - - // apply wait jitter correction ro buffer - rb_jitter_correction(state, outbuf); - - // push buffer downstream - const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); - - if (ret == GST_FLOW_FLUSHING) { - GST_INFO_OBJECT(state->plugin, - "Pad is flushing and does not accept buffers anymore"); - } else if (ret != GST_FLOW_OK) { - GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); - } - - g_mutex_lock(&state->push_queue_mutex); - } - - g_mutex_unlock(&state->push_queue_mutex); - - return NULL; -} -#endif - /** * Send a video buffer to the source pad downstream. * Buffer is checked and timestamps are populated before sending. @@ -767,7 +466,7 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, if (gst_buffer_get_size(outbuf) == 0) { GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); - rb_queue_gl_buffer_cleanup(state, outbuf); + bd_queue_gl_buffer_disposal(&state->buffer_disposal, outbuf); } else { // populate timestamps after rendering so they can't be changed by accident GST_BUFFER_PTS(outbuf) = pts; @@ -776,14 +475,14 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, GstFlowReturn ret; if (state->is_realtime) { -#if PUSH_QUEUE_SIZE > 0 +#if PUSH_BUFFER_ENABLED == 1 // for real-time, we need to wait until it's time to push the buffer // dispatch to queue may block until capacity is available - gboolean result = rb_queue_push_buffer(state, outbuf); + gboolean result = pb_queue_buffer(&state->push_buffer, outbuf); if (result) { ret = GST_FLOW_OK; } else { - rb_queue_gl_buffer_cleanup(state, outbuf); + bd_queue_gl_buffer_disposal(&state->buffer_disposal, outbuf); ret = GST_FLOW_ERROR; } } else { @@ -802,8 +501,8 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, gst_clock_id_wait(clock_id, &jitter); gst_clock_id_unref(clock_id); - rb_calculate_avg_jitter(state, jitter); - rb_jitter_correction(state, outbuf); + pb_calculate_avg_jitter(&state->push_buffer, jitter); + pb_jitter_correction(&state->push_buffer, outbuf); } gst_object_unref(clock); } @@ -817,7 +516,7 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, } else if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } -#if PUSH_QUEUE_SIZE > 0 +#if PUSH_BUFFER_ENABLED == 1 } #endif return ret; @@ -884,6 +583,7 @@ GstFlowReturn rb_render_blocking(RBRenderBuffer *state, GstBuffer *in_audio, /** * Clears all render slots, releases all buffers currently held, resets the * render queue state and EMA state. + * Needs to be called from the GL thread. * * @param state The render buffer to clear. */ @@ -900,7 +600,8 @@ static void rb_clear_slots(RBRenderBuffer *state) { state->slots[i].in_audio = NULL; } if (state->slots[i].out_buf) { - rb_queue_gl_buffer_cleanup(state, state->slots[i].out_buf); + bd_queue_gl_buffer_disposal(&state->buffer_disposal, + state->slots[i].out_buf); state->slots[i].out_buf = NULL; } state->slots[i].state = RB_EMPTY; @@ -1029,22 +730,6 @@ static gpointer rb_render_thread_func(gpointer user_data) { return NULL; } -/** - * Release all buffers currently queued for disposal. - * Needs to be called from GL thread. - * - * @param state Renderbuffer owning cleanup queue to clear. - */ -static void rb_clear_cleanup_queue(RBRenderBuffer *state) { - g_assert(state != NULL); - - // make sure all gl buffers are released - gpointer item; - while ((item = g_async_queue_try_pop(state->buffer_cleanup_queue)) != NULL) { - rb_cb_gl_buffer_cleanup(NULL, item); - } -} - /** * Clears all queues. * Needs to be called from GL thread. @@ -1055,29 +740,23 @@ void rb_clear(RBRenderBuffer *state) { g_assert(state != NULL); rb_clear_slots(state); -#if PUSH_QUEUE_SIZE > 0 - rb_clear_push_queue(state); +#if PUSH_BUFFER_ENABLED == 1 + pb_clear_queue(&state->push_buffer); #endif - rb_clear_cleanup_queue(state); + bd_clear_disposal_queue(&state->buffer_disposal); } -void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, - GstPad *src_pad) { +void rb_start(RBRenderBuffer *state) { g_assert(state != NULL); - state->gl_context = gl_context; - state->src_pad = src_pad; + g_atomic_int_set(&state->running, TRUE); // threads are not needed for offline rendering if (state->is_realtime) { + bd_start_buffer_disposal(&state->buffer_disposal); + pb_start_push_buffer(&state->push_buffer); state->render_thread = g_thread_new("rb-render-thread", rb_render_thread_func, state); -#if PUSH_QUEUE_SIZE > 0 - state->push_thread = - g_thread_new("rb-push-thread", rb_push_thread_func, state); -#endif - state->cleanup_thread = - g_thread_new("rb-cleanup-thread", rb_cleanup_thread_func, state); } GST_INFO_OBJECT(state->plugin, "Started render buffer"); @@ -1095,29 +774,15 @@ void rb_stop(RBRenderBuffer *state) { g_cond_broadcast(&state->render_queued_cond); g_mutex_unlock(&state->slot_lock); + pb_stop_push_buffer(&state->push_buffer); + // wait for render thread to exit g_thread_join(state->render_thread); state->render_thread = NULL; -#if PUSH_QUEUE_SIZE > 0 - // signal wake up push thread to signal loop exit - g_mutex_lock(&state->push_queue_mutex); - g_cond_broadcast(&state->push_queue_cond); - g_cond_broadcast(&state->push_queue_free_cond); - g_mutex_unlock(&state->push_queue_mutex); - // wait for push thread to exit - g_thread_join(state->push_thread); - state->push_thread = NULL; -#endif - - // signal and wait for cleanup thread to exit - g_async_queue_push(state->buffer_cleanup_queue, RB_Q_SHUTDOWN_SIGNAL); - g_thread_join(state->cleanup_thread); - state->cleanup_thread = NULL; + bd_stop_buffer_disposal(&state->buffer_disposal); } - state->gl_context = NULL; - state->src_pad = NULL; GST_INFO_OBJECT(state->plugin, "Stopped render buffer"); } diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 8ef62d4..2d04fe2 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -2,18 +2,9 @@ * Utility to allow offloading of rendering tasks from the plugin chain * function. * - * Functionality provided: - * - A ring buffer based rendering task queue with a limited number of rendering - * slots. It is being consumed by a dedicated thread (render thread) to - * dispatch rendering to the GL thread. - * - * - A ring buffer based queue to schedule GL buffers to be pushed - * downstream at presentation time. It is being consumed by a dedicated thread - * (push thread). - * - * - An async queue to dispose of dropped GL buffers. It is being consumed by - * a dedicated thread (cleanup thread) to dispatch GL buffer cleanup to the GL - * thread. + * A ring buffer based rendering task queue with a limited number of rendering + * slots. It is being consumed by a dedicated thread (render thread) to + * dispatch rendering to the GL thread. * * --- * @@ -55,11 +46,7 @@ * EMA will also recover fps when render performance increases again. * * - GL buffers that completed rendering are scheduled to be pushed to the - * source pad at presentation time (PTS). The queue implemented as a ring - * buffer that blocks on insert in case it is at capacity until the buffer - * can be scheduled. The render loop is throttled by waiting on a free slot. - * A separate worker thread consumes the ring buffer and waits for reaching - * the PTS of the current GL buffer before pushing it downstream. + * source pad at presentation time (PTS) using a PBPushBuffer. */ #ifndef __RENDERBUFFER_H__ @@ -68,6 +55,9 @@ #include #include +#include "bufferdisposal.h" +#include "pushbuffer.h" + G_BEGIN_DECLS /** @@ -87,21 +77,8 @@ G_BEGIN_DECLS #define NUM_RENDER_SLOTS 2 #endif -/** - * Max number of gl frame buffers waiting in a scheduled state to be pushed. - * The push queue decouples the render loop from buffer push timing, allowing - * the render loop to render frames ahead up to the queue capacity. - * Capacity should be low (1-2) to allow back-pressure from fps increases to - * propagate quickly. The queuing call will block when capacity is reached and - * throttle the render loop, frames are never dropped. - * - * 0 : Disable push queuing, block render loop directly until PTS of current - * frame is reached. Disables the push queue API entirely. - * >0 : Allow n buffers waiting in the queue for pushing while render thread - * continues. - */ -#ifndef PUSH_QUEUE_SIZE -#define PUSH_QUEUE_SIZE 1 +#ifndef PUSH_BUFFER_ENABLED +#define PUSH_BUFFER_ENABLED 1 #endif /** @@ -233,49 +210,14 @@ typedef struct { */ GstPad *src_pad; - /** - * Thread running the render loop. - */ - GThread *render_thread; + BDBufferDisposal buffer_disposal; - /** - * Thread running the gl buffer clean-up loop, - * used to release dropped buffer from the gl thread. - */ - GThread *cleanup_thread; + PBPushBuffer push_buffer; /** - * Thread for pushing gl buffers downstream. - * Used for real-time, pushing needs to be scheduled to be synchronized with - * the pipeline clock. - */ - GThread *push_thread; - - /** - * Ring buffer to schedule gl buffers for pushing. - */ - GstBuffer *push_queue[PUSH_QUEUE_SIZE]; - - /** - * Mutex for push ring buffer. - */ - GMutex push_queue_mutex; - - /** - * Condition signaled when a buffer has been queued. - */ - GCond push_queue_cond; - - /** - * Condition signaled when a buffer has been pushed - * and a slot if free. - */ - GCond push_queue_free_cond; - - /** - * Queue to dispose of dropped gl buffers. + * Thread running the render loop. */ - GAsyncQueue *buffer_cleanup_queue; + GThread *render_thread; /** * Callback function pointer to let the plugin know to change fps. @@ -358,32 +300,6 @@ typedef struct { */ guint64 ema_smoothed_render_time; - // concurrent access, protected by push_queue_mutex - // -------------------------------------------------------------- - - /** - * Push ring buffer write position. - */ - gint push_queue_write_idx; - - /** - * Push ring buffer read position. - */ - gint push_queue_read_idx; - - // used only by either render or push thread - // -------------------------------------------------------------- - - /** - * EMA based clock jitter average. - */ - gdouble avg_jitter; - - /** - * Clock jitter initialized. - */ - gboolean avg_jitter_init; - } RBRenderBuffer; /** @@ -428,8 +344,10 @@ typedef struct { * @param max_frame_duration FPS adjustment lower limit. * @param caps_frame_duration FPS requested by pipeline caps. * @param is_qos_enabled Controls if render-time QoS is enabled (EMA). + * @param is_realtime If TRUE async rendering is used. */ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, + GstGLContext *gl_context, GstPad *src_pad, GstGLContextThreadFunc gl_fill_func, RBAdjustFpsFunc adjust_fps_func, GstClockTime max_frame_duration, @@ -473,6 +391,7 @@ void rb_queue_render_task_log(RBQueueArgs *args); * queuing may not be used with the same render buffer at the same time. * * @param state Render buffer to use. + * @param in_audio Audio buffer to pass to projectM. No ownership. * @param pts Frame PTS. * @param frame_duration Frame duration. * @return The downstream push result. @@ -507,10 +426,8 @@ void rb_clear(RBRenderBuffer *state); * Needs to be called from GL thread. * * @param state Render buffer to use. - * @param gl_context GL context to use for rendering. - * @param src_pad Source pad to push video buffers to. */ -void rb_start(RBRenderBuffer *state, GstGLContext *gl_context, GstPad *src_pad); +void rb_start(RBRenderBuffer *state); /** * Stop render loop. Active threads will be joined before returning. From 32e6052ae87b67e5d243e719204e8accaed42070 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 18 Oct 2025 01:31:53 -0500 Subject: [PATCH 23/32] review fixes --- src/bufferdisposal.c | 31 ++++++++++++++++++++++--------- src/bufferdisposal.h | 6 +++--- src/gstglbaseaudiovisualizer.c | 12 ++++++------ src/gstpmaudiovisualizer.c | 12 +++++------- src/pushbuffer.c | 2 +- src/pushbuffer.h | 8 ++++---- 6 files changed, 41 insertions(+), 30 deletions(-) diff --git a/src/bufferdisposal.c b/src/bufferdisposal.c index 879da3a..44246f8 100644 --- a/src/bufferdisposal.c +++ b/src/bufferdisposal.c @@ -9,12 +9,6 @@ GST_DEBUG_CATEGORY_STATIC(buffercleanup_debug); */ static gpointer BC_Q_SHUTDOWN_SIGNAL = &BC_Q_SHUTDOWN_SIGNAL; -void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf) { - g_assert(state != NULL); - g_assert(buf != NULL); - g_async_queue_push(state->disposal_queue, buf); -} - /** * Callback for scheduling gl buffer release with gl thread. * Needs to be called from the GL thread. @@ -24,9 +18,24 @@ void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf) { */ void bd_gl_buffer_dispose_gl(GstGLContext *context, gpointer buf) { (void)context; + + GstGLSyncMeta *sync_meta = gst_buffer_get_gl_sync_meta(buf); + if (sync_meta) + gst_gl_sync_meta_set_sync_point(sync_meta, context); + gst_buffer_unref(GST_BUFFER(buf)); } +void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf) { + g_assert(state != NULL); + g_assert(buf != NULL); + if (gst_gl_context_get_current() == state->gl_context) { + bd_gl_buffer_dispose_gl(state->gl_context, buf); + } else { + g_async_queue_push(state->disposal_queue, buf); + } +} + /** * Used to dispose of dropped gl buffers that are not making it to the src pad. * Consume buffers to clean-up and dispatch release through gl thread. @@ -59,7 +68,7 @@ static gpointer bd_dispose_thread_func(gpointer user_data) { * @param user_data Queue state to use. * @return NULL */ -void bd_clear_queue_gl(GstGLContext* context, gpointer user_data) { +void bd_clear_queue_gl(GstGLContext *context, gpointer user_data) { BDBufferDisposal *state = (BDBufferDisposal *)user_data; g_assert(state != NULL); @@ -71,7 +80,11 @@ void bd_clear_queue_gl(GstGLContext* context, gpointer user_data) { } void bd_clear_disposal_queue(BDBufferDisposal *state) { - gst_gl_context_thread_add(state->gl_context, bd_clear_queue_gl, state); + if (gst_gl_context_get_current() == state->gl_context) { + bd_clear_queue_gl(state->gl_context, state); + } else { + gst_gl_context_thread_add(state->gl_context, bd_clear_queue_gl, state); + } } void bd_init_buffer_disposal(BDBufferDisposal *state, @@ -98,7 +111,7 @@ void bd_start_buffer_disposal(BDBufferDisposal *state) { g_atomic_int_set(&state->running, TRUE); state->disposal_thread = - g_thread_new("rb-cleanup-thread", bd_dispose_thread_func, state); + g_thread_new("bd-disposal-thread", bd_dispose_thread_func, state); } void bd_stop_buffer_disposal(BDBufferDisposal *state) { diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h index b330341..6c37986 100644 --- a/src/bufferdisposal.h +++ b/src/bufferdisposal.h @@ -1,7 +1,7 @@ /* - * An async queue to dispose of dropped GL buffers. It is being consumed by - * a dedicated thread (disposal thread) to dispatch GL buffer cleanup to the GL - * thread. + * An async queue to dispose of (dropped) GL buffers. It is being consumed by + * a dedicated thread (bd-disposal-thread) to dispatch GL buffer cleanup to the + * GL thread. */ #ifndef __BUFFERDISPOSAL_H__ diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 38d07ea..5987d16 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -75,12 +75,12 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); * Allow 0.75 * fps frame duration as wait time for frame render queuing before * dropping previous frame. */ -#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N 3 +#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 3 #endif -#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D 4 +#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 4 #endif /* @@ -622,8 +622,8 @@ static GstFlowReturn gst_gl_base_audio_visualizer_fill( // limit wait based on fps factor, make sure we never wait too long in order // to keep in sync args.max_wait = (GstClockTimeDiff)gst_util_uint64_scale_int( - frame_duration, MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_N, - MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATRIONS_D); + frame_duration, MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N, + MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D); g_rec_mutex_unlock(&glav->priv->context_lock); diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index ddaeaa6..8ce8f4a 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -367,14 +367,12 @@ static void gst_pm_audio_visualizer_dispose(GObject *object) { g_object_unref(scope->priv->adapter); scope->priv->adapter = NULL; } - if (scope->priv->config_lock.p) { - g_mutex_clear(&scope->priv->config_lock); - scope->priv->config_lock.p = NULL; - } - if (scope->priv->ready_cond.p) { - g_cond_clear(&scope->priv->ready_cond); - scope->priv->ready_cond.p = NULL; + if (scope->priv->inbuf) { + gst_buffer_unref(scope->priv->inbuf); + scope->priv->inbuf = NULL; } + g_mutex_clear(&scope->priv->config_lock); + g_cond_clear(&scope->priv->ready_cond); G_OBJECT_CLASS(parent_class)->dispose(object); } diff --git a/src/pushbuffer.c b/src/pushbuffer.c index 74dc318..7d89370 100644 --- a/src/pushbuffer.c +++ b/src/pushbuffer.c @@ -254,7 +254,7 @@ void pb_start_push_buffer(PBPushBuffer *state) { g_atomic_int_set(&state->running, TRUE); state->push_thread = - g_thread_new("rb-push-thread", rb_push_thread_func, state); + g_thread_new("pb-push-thread", rb_push_thread_func, state); } void pb_stop_push_buffer(PBPushBuffer *state) { diff --git a/src/pushbuffer.h b/src/pushbuffer.h index a7245ec..fd50b60 100644 --- a/src/pushbuffer.h +++ b/src/pushbuffer.h @@ -1,9 +1,9 @@ /* * A ring buffer based queue to schedule GL buffers to be pushed - * downstream at presentation time (PTS). It is being consumed by a dedicated - * thread (push thread) used for processing. The queuing call will block when - * capacity is reached and throttle the render loop by letting it wait for a - * free slot. Frames are never dropped. + * downstream at presentation time (PTS). The queue is consumed by a dedicated + * thread (pb-push-thread) to wait for the next scheduled push. The queuing call + * will block when capacity is reached and throttle the render loop by letting + * it wait for a free slot. Frames are never dropped. */ #ifndef __PUSHBUFFER_H__ From 69387d96367dfac92b41145b19304639f1ce4bb1 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 18 Oct 2025 01:55:08 -0500 Subject: [PATCH 24/32] tune params --- src/gstglbaseaudiovisualizer.c | 4 ++-- src/pushbuffer.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 5987d16..564076b 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -76,11 +76,11 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); * dropping previous frame. */ #ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 3 +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 2 #endif #ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 4 +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 3 #endif /* diff --git a/src/pushbuffer.c b/src/pushbuffer.c index 7d89370..274d60e 100644 --- a/src/pushbuffer.c +++ b/src/pushbuffer.c @@ -10,7 +10,7 @@ GST_DEBUG_CATEGORY_STATIC(pushbuffer_debug); * EMA aloha for push schedule clock jitter average. */ #ifndef JITTER_EMA_ALPHA -#define JITTER_EMA_ALPHA 0.75 +#define JITTER_EMA_ALPHA 0.85 #endif /** From f5af382ce661b6144931df925f750b66baf95aeb Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sat, 18 Oct 2025 17:51:55 -0500 Subject: [PATCH 25/32] review fixes --- README.md | 19 ++-- src/bufferdisposal.c | 59 +++++++++---- src/bufferdisposal.h | 11 +-- src/pushbuffer.c | 202 +++++++++++++++++++++++++++---------------- src/pushbuffer.h | 54 ++++++++---- src/renderbuffer.c | 137 +++++++++++++++-------------- src/renderbuffer.h | 22 ++--- 7 files changed, 307 insertions(+), 197 deletions(-) diff --git a/README.md b/README.md index dd8154d..acb5951 100644 --- a/README.md +++ b/README.md @@ -276,15 +276,16 @@ pipeline caps FPS. Video frame PTS offset is derived from the **first audio buffer PTS** or **segment event** plus accumulated samples to align with audio timing. -| Timing Source | Origin | Applies to clock | Purpose | -|----------------------------------|--------------------|------------------|---------------------------------------------------------------------------------------------------| -| Audio Timestamps | Audio Input | Always | Determine video timing and sync. | -| Sample Rate / Pipeline FPS | Audio Input / Caps | Always | Defines how many audio samples are used per frame and target FPS. | -| Segment Info | Segment Event | Always | Tracks running time and playback position. Used for PTS offsets. | -| QoS Feedback | QoS Event | Live | Skips outdated frames to reduce latency. | -| Render Frame Drop | Render Loop | Live | Drop frames that cannot be rendered in time. | -| Exponential Moving Average (EMA) | Render Loop | Live | Adjust plugin target FPS in case frame render time exceeds the real-time budget most of the time. | -| Latency Event | Render Loop | Live | Inform upstream of latency changes in case of adaptive FPS changes (EMA). | +| Timing Source | Origin | Applies to clock | Purpose | +|----------------------------|--------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Audio Timestamps | Audio Input | Always | Determine video timing and sync. | +| Sample Rate / Pipeline FPS | Audio Input / Caps | Always | Defines how many audio samples are used per frame and target FPS. | +| Segment Info | Segment Event | Always | Tracks running time and playback position. Used for PTS offsets. | +| QoS Feedback | QoS Event | Live | Skips outdated frames to correct sync with downstream sink/pipeline clock. | +| Render Frame Drop | Render Loop | Live | Drop frames that cannot be rendered in time to keep sync with pipeline clock. | +| GL Frame Render Duration | Render Loop | Live | Exponential Moving Average of the frame render duration. Adjusts plugin target FPS in case exceeds the real-time budget most of the time. | +| Latency Event | Render Loop | Live | Inform upstream of latency changes in case of adaptive FPS changes (via EMA). | +| Buffer push clock jitter | Render Loop | Live | Exponential Moving Average of the source pad push jitter caused by the scheduler. Clocks in gstreamer are not guaranteed to be precise with timed waits, as this cannot be guaranteed by the operating system. Adds jitter EMA as a correction to the buffer PTS. | --- diff --git a/src/bufferdisposal.c b/src/bufferdisposal.c index 44246f8..716106f 100644 --- a/src/bufferdisposal.c +++ b/src/bufferdisposal.c @@ -7,7 +7,7 @@ GST_DEBUG_CATEGORY_STATIC(buffercleanup_debug); /** * Queue shutdown signal token. */ -static gpointer BC_Q_SHUTDOWN_SIGNAL = &BC_Q_SHUTDOWN_SIGNAL; +static gpointer BD_Q_SHUTDOWN_SIGNAL = &BD_Q_SHUTDOWN_SIGNAL; /** * Callback for scheduling gl buffer release with gl thread. @@ -17,16 +17,21 @@ static gpointer BC_Q_SHUTDOWN_SIGNAL = &BC_Q_SHUTDOWN_SIGNAL; * @param buf GL buffer to release. */ void bd_gl_buffer_dispose_gl(GstGLContext *context, gpointer buf) { - (void)context; - GstGLSyncMeta *sync_meta = gst_buffer_get_gl_sync_meta(buf); - if (sync_meta) - gst_gl_sync_meta_set_sync_point(sync_meta, context); + GstBuffer *buffer = GST_BUFFER(buf); + if (buffer == NULL) + return; - gst_buffer_unref(GST_BUFFER(buf)); + if (context != NULL) { + GstGLSyncMeta *sync_meta = gst_buffer_get_gl_sync_meta(buffer); + if (sync_meta) + gst_gl_sync_meta_set_sync_point(sync_meta, context); + } + + gst_buffer_unref(GST_BUFFER(buffer)); } -void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf) { +void bd_dispose_gl_buffer(BDBufferDisposal *state, GstBuffer *buf) { g_assert(state != NULL); g_assert(buf != NULL); if (gst_gl_context_get_current() == state->gl_context) { @@ -43,7 +48,7 @@ void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf) { * @param user_data Queue state to use. * @return NULL */ -static gpointer bd_dispose_thread_func(gpointer user_data) { +static gpointer _bd_dispose_thread_func(gpointer user_data) { BDBufferDisposal *state = (BDBufferDisposal *)user_data; g_assert(state != NULL); @@ -53,11 +58,15 @@ static gpointer bd_dispose_thread_func(gpointer user_data) { gpointer item = g_async_queue_pop(state->disposal_queue); - if (!item || item == BC_Q_SHUTDOWN_SIGNAL) + if (item == BD_Q_SHUTDOWN_SIGNAL) + break; + + if (!item) continue; gst_gl_context_thread_add(state->gl_context, bd_gl_buffer_dispose_gl, item); } + return NULL; } @@ -71,6 +80,7 @@ static gpointer bd_dispose_thread_func(gpointer user_data) { void bd_clear_queue_gl(GstGLContext *context, gpointer user_data) { BDBufferDisposal *state = (BDBufferDisposal *)user_data; g_assert(state != NULL); + g_assert(gst_gl_context_get_current() == context); // make sure all gl buffers are released gpointer item; @@ -79,7 +89,9 @@ void bd_clear_queue_gl(GstGLContext *context, gpointer user_data) { } } -void bd_clear_disposal_queue(BDBufferDisposal *state) { +void bd_clear(BDBufferDisposal *state) { + g_assert(state != NULL); + if (gst_gl_context_get_current() == state->gl_context) { bd_clear_queue_gl(state->gl_context, state); } else { @@ -89,9 +101,13 @@ void bd_clear_disposal_queue(BDBufferDisposal *state) { void bd_init_buffer_disposal(BDBufferDisposal *state, GstGLContext *gl_context) { + g_assert(state != NULL); - GST_DEBUG_CATEGORY_INIT(buffercleanup_debug, "buffercleanup", 0, - "projectM visualizer plugin buffer cleanup"); + static gsize _debug_initialized = 0; + if (g_once_init_enter(&_debug_initialized)) { + GST_DEBUG_CATEGORY_INIT(buffercleanup_debug, "buffercleanup", 0, + "projectM visualizer plugin buffer cleanup"); + } // init clean up queue state->disposal_thread = NULL; @@ -101,24 +117,33 @@ void bd_init_buffer_disposal(BDBufferDisposal *state, } void bd_dispose_buffer_disposal(BDBufferDisposal *state) { + g_assert(state != NULL); + g_assert(state->disposal_thread == NULL); g_async_queue_unref(state->disposal_queue); state->disposal_queue = NULL; state->gl_context = NULL; } void bd_start_buffer_disposal(BDBufferDisposal *state) { + g_assert(state != NULL); g_atomic_int_set(&state->running, TRUE); - state->disposal_thread = - g_thread_new("bd-disposal-thread", bd_dispose_thread_func, state); + if (state->disposal_thread == NULL) { + state->disposal_thread = + g_thread_new("bd-disposal-thread", _bd_dispose_thread_func, state); + } } void bd_stop_buffer_disposal(BDBufferDisposal *state) { + g_assert(state != NULL); + // signal and wait for cleanup thread to exit g_atomic_int_set(&state->running, FALSE); - g_async_queue_push(state->disposal_queue, BC_Q_SHUTDOWN_SIGNAL); - g_thread_join(state->disposal_thread); - state->disposal_thread = NULL; + if (state->disposal_thread != NULL) { + g_async_queue_push(state->disposal_queue, BD_Q_SHUTDOWN_SIGNAL); + g_thread_join(state->disposal_thread); + state->disposal_thread = NULL; + } } \ No newline at end of file diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h index 6c37986..5dd0f48 100644 --- a/src/bufferdisposal.h +++ b/src/bufferdisposal.h @@ -1,6 +1,6 @@ /* - * An async queue to dispose of (dropped) GL buffers. It is being consumed by - * a dedicated thread (bd-disposal-thread) to dispatch GL buffer cleanup to the + * An async queue to dispose of (dropped) GL buffers. The queue is consumed by + * a dedicated thread (bd-disposal-thread) to dispatch GL buffer unref to the * GL thread. */ @@ -42,19 +42,20 @@ typedef struct { } BDBufferDisposal; /** - * Release given buffer from the GL thread. + * Dispose given buffer from the GL thread. + * Disposal will be queued if current thread is not the GL thread. * * @param state State to use. * @param buf Buffer to dispose. */ -void bd_queue_gl_buffer_disposal(BDBufferDisposal *state, GstBuffer *buf); +void bd_dispose_gl_buffer(BDBufferDisposal *state, GstBuffer *buf); /** * Dispose of all buffers currently queued for disposal. * * @param state Renderbuffer owning cleanup queue to clear. */ -void bd_clear_disposal_queue(BDBufferDisposal *state); +void bd_clear(BDBufferDisposal *state); /** * Init queue state. diff --git a/src/pushbuffer.c b/src/pushbuffer.c index 274d60e..1cec410 100644 --- a/src/pushbuffer.c +++ b/src/pushbuffer.c @@ -1,6 +1,8 @@ #include "pushbuffer.h" +#include + #include "bufferdisposal.h" GST_DEBUG_CATEGORY_STATIC(pushbuffer_debug); @@ -14,7 +16,8 @@ GST_DEBUG_CATEGORY_STATIC(pushbuffer_debug); #endif /** - * EMA for push schedule clock jitter outlier threshold. + * Push schedule clock jitter outlier threshold, any jitter duration above this + * value will be ignored. */ #ifndef JITTER_EMA_OUTLIER_THRESHOLD #define JITTER_EMA_OUTLIER_THRESHOLD (5 * GST_MSECOND) @@ -35,29 +38,28 @@ gboolean pb_queue_buffer(PBPushBuffer *state, GstBuffer *buffer) { g_mutex_lock(&state->push_queue_mutex); - // write to the next position in the ring - state->push_queue_write_idx = - (state->push_queue_write_idx + 1) % PUSH_QUEUE_SIZE; - - gboolean result = TRUE; + gint next_idx = (state->push_queue_write_idx + 1) % PUSH_QUEUE_SIZE; // wait until next position is free - while (state->push_queue[state->push_queue_write_idx] != NULL) { + while (state->push_queue[next_idx] != NULL && + g_atomic_int_get(&state->running)) { g_cond_wait(&state->push_queue_free_cond, &state->push_queue_mutex); - if (!g_atomic_int_get(&state->running)) { - result = FALSE; - break; - } + next_idx = (state->push_queue_write_idx + 1) % PUSH_QUEUE_SIZE; } - if (result) { + gboolean ret = FALSE; + if (g_atomic_int_get(&state->running)) { + // write to the next position in the ring + state->push_queue_write_idx = next_idx; + g_assert(state->push_queue[state->push_queue_write_idx] == NULL); // take the spot and signal that queue has changed state->push_queue[state->push_queue_write_idx] = buffer; g_cond_signal(&state->push_queue_cond); + ret = TRUE; } g_mutex_unlock(&state->push_queue_mutex); - return result; + return ret; } void pb_calculate_avg_jitter(PBPushBuffer *state, @@ -67,27 +69,70 @@ void pb_calculate_avg_jitter(PBPushBuffer *state, return; if (!state->avg_jitter_init) { - state->avg_jitter = (gdouble)jitter; + state->avg_jitter = jitter; state->avg_jitter_init = TRUE; } else { - state->avg_jitter = JITTER_EMA_ALPHA * state->avg_jitter + - (1.0 - JITTER_EMA_ALPHA) * (gdouble)jitter; + gdouble v = (1.0 - JITTER_EMA_ALPHA) * (gdouble)state->avg_jitter + + JITTER_EMA_ALPHA * (gdouble)jitter; + state->avg_jitter = (GstClockTimeDiff)llround(v); } } -static void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf) { +void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf) { if (GST_BUFFER_PTS(outbuf) != GST_CLOCK_TIME_NONE) { - GstClockTime correction = llabs((guint64)state->avg_jitter); + GstClockTime correction = llabs(state->avg_jitter); - if (state->avg_jitter > 0.0) { + if (state->avg_jitter > 0 && GST_BUFFER_PTS(outbuf) > correction) { GST_BUFFER_PTS(outbuf) -= correction; - } else { + } else if (state->avg_jitter < 0) { GST_BUFFER_PTS(outbuf) += correction; } } } +GstClockReturn pb_wait_to_push(PBPushBuffer *state, const GstClockTime pts) { + GstClockReturn ret = GST_CLOCK_UNSUPPORTED; + + if (state->clock) { + const GstClockTime base_time = + gst_element_get_base_time(GST_ELEMENT(state->plugin)); + + if (base_time == GST_CLOCK_TIME_NONE) { + return GST_CLOCK_UNSCHEDULED; + } + + const GstClockTime abs_time = pts + base_time; + + const GstClockTimeDiff remaining_wait = + GST_CLOCK_DIFF(abs_time, gst_clock_get_time(state->clock)); + + if (remaining_wait < MIN_PUSH_SCHEDULE_WAIT) { + // we don't need to wait, all good + return GST_CLOCK_OK; + } + + // we need to wait + GstClockTimeDiff jitter = 0; + const GstClockID clock_id = + gst_clock_new_single_shot_id(state->clock, abs_time); + + ret = gst_clock_id_wait(clock_id, &jitter); + + gst_clock_id_unref(clock_id); + + if (ret == GST_CLOCK_OK || ret == GST_CLOCK_EARLY) { + // record jitter + pb_calculate_avg_jitter(state, jitter); + } + GST_TRACE_OBJECT(state->plugin, + "Push jitter avg=%" G_GINT64_FORMAT " ns, ret=%d", + state->avg_jitter, ret); + } + + return ret; +} + /** * Consume buffers to push and wait until it's PTS time to push. * Used for real-time rendering only. @@ -96,7 +141,7 @@ static void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf) { * @param user_data Render buffer to use. * @return NULL */ -static gpointer rb_push_thread_func(gpointer user_data) { +static gpointer _rb_push_thread_func(gpointer user_data) { PBPushBuffer *state = (PBPushBuffer *)user_data; g_assert(state != NULL); @@ -105,68 +150,47 @@ static gpointer rb_push_thread_func(gpointer user_data) { while (g_atomic_int_get(&state->running)) { - state->push_queue_read_idx = - (state->push_queue_read_idx + 1) % PUSH_QUEUE_SIZE; - // consume gl buffer to push gboolean stop = FALSE; - while (state->push_queue[state->push_queue_read_idx] == NULL) { + gint next_idx = (state->push_queue_read_idx + 1) % PUSH_QUEUE_SIZE; + while (state->push_queue[next_idx] == NULL) { // no buffer to push, wait for one g_cond_wait(&state->push_queue_cond, &state->push_queue_mutex); if (!g_atomic_int_get(&state->running)) { stop = TRUE; break; } + next_idx = (state->push_queue_read_idx + 1) % PUSH_QUEUE_SIZE; } if (stop) { break; } + state->push_queue_read_idx = next_idx; + // found a buffer to push GstBuffer *outbuf = state->push_queue[state->push_queue_read_idx]; + g_assert(outbuf != NULL); + // determine when it's time to push const GstClockTime pts = GST_BUFFER_PTS(outbuf); + GstClockReturn clock_return = GST_CLOCK_UNSUPPORTED; if (pts != GST_CLOCK_TIME_NONE) { - GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - const GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - - const GstClockTime abs_time = pts + base_time; - - const GstClockTimeDiff remaining_wait = - GST_CLOCK_DIFF(gst_clock_get_time(clock), abs_time); - - if (remaining_wait > MIN_PUSH_SCHEDULE_WAIT) { - // we need to wait, unlock first - g_mutex_unlock(&state->push_queue_mutex); - - GstClockTimeDiff jitter = 0; - const GstClockID clock_id = - gst_clock_new_single_shot_id(clock, abs_time); - const GstClockReturn clock_return = - gst_clock_id_wait(clock_id, &jitter); - gst_clock_id_unref(clock_id); - - if (clock_return == GST_CLOCK_OK || clock_return == GST_CLOCK_EARLY) { - // record jitter - pb_calculate_avg_jitter(state, jitter); - } else if (clock_return == GST_CLOCK_UNSCHEDULED) { - // drop buffer if clock is not running - g_mutex_lock(&state->push_queue_mutex); - if (state->push_queue[state->push_queue_read_idx] == outbuf) { - state->push_queue[state->push_queue_read_idx] = NULL; - bd_queue_gl_buffer_disposal(state->buffer_disposal, outbuf); - } - gst_object_unref(clock); - continue; - } - g_mutex_lock(&state->push_queue_mutex); + + g_mutex_unlock(&state->push_queue_mutex); + clock_return = pb_wait_to_push(state, pts); + g_mutex_lock(&state->push_queue_mutex); + + if (clock_return == GST_CLOCK_UNSCHEDULED) { + // drop buffer if clock is not running + if (state->push_queue[state->push_queue_read_idx] == outbuf) { + state->push_queue[state->push_queue_read_idx] = NULL; + bd_dispose_gl_buffer(state->buffer_disposal, outbuf); } + continue; } - gst_object_unref(clock); } // now we own the buffer to push @@ -174,8 +198,10 @@ static gpointer rb_push_thread_func(gpointer user_data) { g_cond_signal(&state->push_queue_free_cond); g_mutex_unlock(&state->push_queue_mutex); - // apply wait jitter correction ro buffer - pb_jitter_correction(state, outbuf); + if (clock_return != GST_CLOCK_UNSUPPORTED) { + // apply wait jitter correction ro buffer + pb_jitter_correction(state, outbuf); + } // push buffer downstream const GstFlowReturn ret = gst_pad_push(state->src_pad, outbuf); @@ -198,8 +224,11 @@ static gpointer rb_push_thread_func(gpointer user_data) { void pb_init_push_buffer(PBPushBuffer *state, BDBufferDisposal *buffer_cleanup, GstObject *plugin, GstPad *src_pad) { - GST_DEBUG_CATEGORY_INIT(pushbuffer_debug, "pushbuffer", 0, - "projectM visualizer plugin push buffer"); + static gsize _debug_initialized = 0; + if (g_once_init_enter(&_debug_initialized)) { + GST_DEBUG_CATEGORY_INIT(pushbuffer_debug, "pushbuffer", 0, + "projectM visualizer plugin push buffer"); + } state->buffer_disposal = buffer_cleanup; state->plugin = plugin; @@ -207,8 +236,8 @@ void pb_init_push_buffer(PBPushBuffer *state, BDBufferDisposal *buffer_cleanup, // init push queue state->push_thread = NULL; - state->push_queue_read_idx = -1; - state->push_queue_write_idx = -1; + state->push_queue_read_idx = PUSH_QUEUE_SIZE - 1; + state->push_queue_write_idx = PUSH_QUEUE_SIZE - 1; g_mutex_init(&state->push_queue_mutex); g_cond_init(&state->push_queue_cond); g_cond_init(&state->push_queue_free_cond); @@ -218,6 +247,8 @@ void pb_init_push_buffer(PBPushBuffer *state, BDBufferDisposal *buffer_cleanup, state->avg_jitter = 0.0; state->avg_jitter_init = FALSE; + + state->clock = NULL; } void pb_dispose_push_buffer(PBPushBuffer *state) { @@ -229,6 +260,11 @@ void pb_dispose_push_buffer(PBPushBuffer *state) { state->buffer_disposal = NULL; state->plugin = NULL; state->src_pad = NULL; + + if (state->clock != NULL) { + gst_object_unref(state->clock); + state->clock = NULL; + } } void pb_clear_queue(PBPushBuffer *state) { @@ -238,12 +274,12 @@ void pb_clear_queue(PBPushBuffer *state) { // release buffers that are still queued before cleanup thread shuts down for (guint i = 0; i < PUSH_QUEUE_SIZE; i++) { if (state->push_queue[i] != NULL) { - bd_queue_gl_buffer_disposal(state->buffer_disposal, state->push_queue[i]); + bd_dispose_gl_buffer(state->buffer_disposal, state->push_queue[i]); state->push_queue[i] = NULL; } } - state->push_queue_read_idx = -1; - state->push_queue_write_idx = -1; + state->push_queue_read_idx = PUSH_QUEUE_SIZE - 1; + state->push_queue_write_idx = PUSH_QUEUE_SIZE - 1; state->avg_jitter = 0.0; state->avg_jitter_init = FALSE; @@ -251,21 +287,37 @@ void pb_clear_queue(PBPushBuffer *state) { } void pb_start_push_buffer(PBPushBuffer *state) { + + if (state->clock != NULL) { + gst_object_unref(state->clock); + } + state->clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); + g_atomic_int_set(&state->running, TRUE); - state->push_thread = - g_thread_new("pb-push-thread", rb_push_thread_func, state); + if (state->push_thread == NULL) { + state->push_thread = + g_thread_new("pb-push-thread", _rb_push_thread_func, state); + } } void pb_stop_push_buffer(PBPushBuffer *state) { g_atomic_int_set(&state->running, FALSE); - // signal wake up push thread to signal loop exit + // signal wake up to conditions that may be blocking g_mutex_lock(&state->push_queue_mutex); g_cond_broadcast(&state->push_queue_cond); g_cond_broadcast(&state->push_queue_free_cond); g_mutex_unlock(&state->push_queue_mutex); + // wait for push thread to exit - g_thread_join(state->push_thread); - state->push_thread = NULL; + if (state->push_thread) { + g_thread_join(state->push_thread); + state->push_thread = NULL; + } + + if (state->clock != NULL) { + gst_object_unref(state->clock); + state->clock = NULL; + } } diff --git a/src/pushbuffer.h b/src/pushbuffer.h index fd50b60..3d2ea54 100644 --- a/src/pushbuffer.h +++ b/src/pushbuffer.h @@ -1,9 +1,11 @@ /* * A ring buffer based queue to schedule GL buffers to be pushed - * downstream at presentation time (PTS). The queue is consumed by a dedicated - * thread (pb-push-thread) to wait for the next scheduled push. The queuing call - * will block when capacity is reached and throttle the render loop by letting - * it wait for a free slot. Frames are never dropped. + * downstream at presentation time (PTS). The push queue decouples the render + * loop from buffer push timing, allowing the render loop to render frames ahead + * up to the queue capacity. The queue is consumed by a dedicated thread + * (pb-push-thread) to wait for the next scheduled push. The queuing call will + * block when capacity is reached until a free slot is available, throttling the + * render loop. Frames are never dropped. */ #ifndef __PUSHBUFFER_H__ @@ -15,13 +17,12 @@ /** * Max number of gl frame buffers waiting in a scheduled state to be pushed. - * The push queue decouples the render loop from buffer push timing, allowing - * the render loop to render frames ahead up to the queue capacity. * Capacity should be low (1-2) to allow back-pressure from fps increases to * propagate quickly. * * 0 : Disable push queuing, block render loop directly until PTS of current - * frame is reached. Disables the push queue API entirely. + * frame is reached. + * * >0 : Allow n buffers waiting in the queue for pushing while render thread * continues. */ @@ -30,11 +31,11 @@ #endif /** - * All render buffer data. + * Push buffer state. */ typedef struct { - // not re-assigned during render thread lifetime + // not re-assigned during push buffer lifetime // -------------------------------------------------------------- /** @@ -76,10 +77,15 @@ typedef struct { /** * Condition signaled when a buffer has been pushed - * and a slot if free. + * and a slot is free. */ GCond push_queue_free_cond; + /** + * Clock to use for scheduling. + */ + GstClock *clock; + // concurrent access, g_atomic // -------------------------------------------------------------- @@ -101,13 +107,13 @@ typedef struct { */ gint push_queue_read_idx; - // used only by either render or push thread + // used only by either render (offline) or push thread (real-time) // -------------------------------------------------------------- /** * EMA based clock jitter average. */ - gdouble avg_jitter; + GstClockTimeDiff avg_jitter; /** * Clock jitter initialized. @@ -117,12 +123,12 @@ typedef struct { } PBPushBuffer; /** - * Schedule a rendered gl buffer for pushing downstream. + * Schedule a rendered gl buffer for getting pushed downstream. * The buffer will not be pushed until it's PTS time is reached. - * This call will block until the buffer can be schduled or + * This call will block until the buffer can be scheduled or * the render buffer is stopped. * - * @param state The render buffer to use. + * @param state The push buffer to use. * @param buffer The gl buffer to push. Takes ownership of the buffer. * * @return TRUE if the buffer was scheduled successfully, FALSE in case the @@ -140,19 +146,31 @@ void pb_clear_queue(PBPushBuffer *state); /** * Applies jitter correction to the given buffer. * - * @param state Current render buffer. + * @param state Current push buffer. * @param outbuf Buffer to apply correction to. */ -static void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf); +void pb_jitter_correction(PBPushBuffer *state, GstBuffer *outbuf); /** * Calculate current clock jitter average. * - * @param state Current render buffer. + * @param state Current push buffer. * @param jitter Latest jitter value. */ void pb_calculate_avg_jitter(PBPushBuffer *state, GstClockTimeDiff jitter); +/** + * Wait until reaching the given PTS. The clock jitter is recorded if it's + * appropriate to do so. + * + * @param state Push buffer to use. + * @param pts Wait until this PTS is reached. + * @param locked Controls if unlocking is needed before entering wait. + * + * @return The clock wait result. + */ +GstClockReturn pb_wait_to_push(PBPushBuffer *state, GstClockTime pts); + /** * Init this push buffer. * diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 41f8119..7a49911 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -85,6 +85,18 @@ GST_DEBUG_CATEGORY_STATIC(renderbuffer_debug); #define MIN_FREE_SLOT_SCHEDULE_WAIT GST_MSECOND #endif +/** + * Controls if pushing of gl buffers is done by the render loop (blocking) + * directly, or deferred to the push buffer. Deferring allows a more responsive + * render timing. + * + * 0 : Push blocks render loop. + * 1 : Push is deferred to push buffer, may or may not block. + */ +#ifndef PUSH_BUFFER_ENABLED +#define PUSH_BUFFER_ENABLED 1 +#endif + /** * Exponential Moving Average (EMA)-based adaptive frame duration (fps) * adjustment. Determines desired frame duration change based on the @@ -176,9 +188,18 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, const gboolean is_realtime) { g_assert(state != NULL); - - GST_DEBUG_CATEGORY_INIT(renderbuffer_debug, "renderbuffer", 0, - "projectM visualizer plugin render buffer"); + g_assert(plugin != NULL); + g_assert(gl_context != NULL); + g_assert(src_pad != NULL); + g_assert(gl_fill_func != NULL); + g_assert(adjust_fps_func != NULL); + g_assert(max_frame_duration >= caps_frame_duration); + + static gsize _debug_initialized = 0; + if (g_once_init_enter(&_debug_initialized)) { + GST_DEBUG_CATEGORY_INIT(renderbuffer_debug, "renderbuffer", 0, + "projectM visualizer plugin render buffer"); + } // context config without ownership state->plugin = plugin; @@ -197,8 +218,8 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, // init render queue state->render_thread = NULL; - state->render_write_idx = -1; - state->render_read_idx = -1; + state->render_write_idx = NUM_RENDER_SLOTS - 1; + state->render_read_idx = NUM_RENDER_SLOTS - 1; g_mutex_init(&state->slot_lock); g_cond_init(&state->slot_available_cond); g_cond_init(&state->render_queued_cond); @@ -310,7 +331,7 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { } if (slot->out_buf != NULL) { - bd_queue_gl_buffer_disposal(&state->buffer_disposal, slot->out_buf); + bd_dispose_gl_buffer(&state->buffer_disposal, slot->out_buf); slot->out_buf = NULL; } @@ -383,7 +404,8 @@ static GstClockTime rb_element_render_time(GstElement *element) { const GstClockTime base_time = gst_element_get_base_time(element); GstClock *clock = gst_element_get_clock(element); const GstClockTime now = gst_clock_get_time(clock); - return now - base_time; + gst_object_unref(clock); + return GST_CLOCK_DIFF(base_time, now); } gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, @@ -466,63 +488,52 @@ static GstFlowReturn rb_handle_push_buffer(RBRenderBuffer *state, if (gst_buffer_get_size(outbuf) == 0) { GST_WARNING_OBJECT(state->plugin, "Empty or invalid buffer, dropping."); - bd_queue_gl_buffer_disposal(&state->buffer_disposal, outbuf); - } else { - // populate timestamps after rendering so they can't be changed by accident - GST_BUFFER_PTS(outbuf) = pts; - GST_BUFFER_DTS(outbuf) = pts; - GST_BUFFER_DURATION(outbuf) = frame_duration; + bd_dispose_gl_buffer(&state->buffer_disposal, outbuf); + return GST_FLOW_OK; + } - GstFlowReturn ret; - if (state->is_realtime) { + // populate timestamps after rendering so they can't be changed by accident + GST_BUFFER_PTS(outbuf) = pts; + GST_BUFFER_DTS(outbuf) = pts; + GST_BUFFER_DURATION(outbuf) = frame_duration; + + GstFlowReturn ret; + if (state->is_realtime) { #if PUSH_BUFFER_ENABLED == 1 - // for real-time, we need to wait until it's time to push the buffer - // dispatch to queue may block until capacity is available - gboolean result = pb_queue_buffer(&state->push_buffer, outbuf); - if (result) { - ret = GST_FLOW_OK; - } else { - bd_queue_gl_buffer_disposal(&state->buffer_disposal, outbuf); - ret = GST_FLOW_ERROR; - } + // for real-time, we need to wait until it's time to push the buffer + // dispatch to queue may block until capacity is available + gboolean result = pb_queue_buffer(&state->push_buffer, outbuf); + if (result) { + ret = GST_FLOW_OK; } else { + bd_dispose_gl_buffer(&state->buffer_disposal, outbuf); + ret = GST_FLOW_ERROR; + } + } else { #else - // blocking wait until buffer has been pushed in time - // then push directly - GstClock *clock = gst_element_get_clock(GST_ELEMENT(state->plugin)); - if (clock) { - const GstClockTime base_time = - gst_element_get_base_time(GST_ELEMENT(state->plugin)); - - const GstClockTime abs_time = pts + base_time; - GstClockTimeDiff jitter = 0; - - GstClockID clock_id = gst_clock_new_single_shot_id(clock, abs_time); - gst_clock_id_wait(clock_id, &jitter); - gst_clock_id_unref(clock_id); - - pb_calculate_avg_jitter(&state->push_buffer, jitter); - pb_jitter_correction(&state->push_buffer, outbuf); - } - gst_object_unref(clock); + // blocking wait until it's time to push the buffer, + // then push directly + const GstClockReturn clock_return = + pb_wait_to_push(&state->push_buffer, pts); + if (clock_return != GST_CLOCK_UNSUPPORTED) { + // apply wait jitter correction ro buffer + pb_jitter_correction(&state->push_buffer, outbuf); } + } #endif - // push buffer downstream directly for offline rendering or if - // queuing is disabled - ret = gst_pad_push(state->src_pad, outbuf); - if (ret == GST_FLOW_FLUSHING) { - GST_INFO_OBJECT(state->plugin, - "Pad is flushing and does not accept buffers anymore"); - } else if (ret != GST_FLOW_OK) { - GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); - } -#if PUSH_BUFFER_ENABLED == 1 + // push buffer downstream directly for offline rendering or if + // queuing is disabled + ret = gst_pad_push(state->src_pad, outbuf); + if (ret == GST_FLOW_FLUSHING) { + GST_INFO_OBJECT(state->plugin, + "Pad is flushing and does not accept buffers anymore"); + } else if (ret != GST_FLOW_OK) { + GST_WARNING_OBJECT(state->plugin, "Failed to push buffer to pad"); } +#if PUSH_BUFFER_ENABLED == 1 + } // endif(state->is_realtime) #endif - return ret; - } - - return GST_FLOW_OK; + return ret; } /** @@ -600,16 +611,15 @@ static void rb_clear_slots(RBRenderBuffer *state) { state->slots[i].in_audio = NULL; } if (state->slots[i].out_buf) { - bd_queue_gl_buffer_disposal(&state->buffer_disposal, - state->slots[i].out_buf); + bd_dispose_gl_buffer(&state->buffer_disposal, state->slots[i].out_buf); state->slots[i].out_buf = NULL; } state->slots[i].state = RB_EMPTY; } } - state->render_write_idx = -1; - state->render_read_idx = -1; + state->render_write_idx = NUM_RENDER_SLOTS - 1; + state->render_read_idx = NUM_RENDER_SLOTS - 1; state->ema_frame_counter = 0; state->ema_smoothed_render_time = 0; @@ -622,7 +632,7 @@ static void rb_clear_slots(RBRenderBuffer *state) { * @param user_data Render buffer to work on. * @return NULL */ -static gpointer rb_render_thread_func(gpointer user_data) { +static gpointer _rb_render_thread_func(gpointer user_data) { RBRenderBuffer *state = (RBRenderBuffer *)user_data; g_assert(state != NULL); @@ -743,7 +753,7 @@ void rb_clear(RBRenderBuffer *state) { #if PUSH_BUFFER_ENABLED == 1 pb_clear_queue(&state->push_buffer); #endif - bd_clear_disposal_queue(&state->buffer_disposal); + bd_clear(&state->buffer_disposal); } void rb_start(RBRenderBuffer *state) { @@ -756,7 +766,7 @@ void rb_start(RBRenderBuffer *state) { bd_start_buffer_disposal(&state->buffer_disposal); pb_start_push_buffer(&state->push_buffer); state->render_thread = - g_thread_new("rb-render-thread", rb_render_thread_func, state); + g_thread_new("rb-render-thread", _rb_render_thread_func, state); } GST_INFO_OBJECT(state->plugin, "Started render buffer"); @@ -772,6 +782,7 @@ void rb_stop(RBRenderBuffer *state) { // wake up render thread to signal loop exit g_mutex_lock(&state->slot_lock); g_cond_broadcast(&state->render_queued_cond); + g_cond_broadcast(&state->slot_available_cond); g_mutex_unlock(&state->slot_lock); pb_stop_push_buffer(&state->push_buffer); diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 2d04fe2..61764b1 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -2,9 +2,9 @@ * Utility to allow offloading of rendering tasks from the plugin chain * function. * - * A ring buffer based rendering task queue with a limited number of rendering - * slots. It is being consumed by a dedicated thread (render thread) to - * dispatch rendering to the GL thread. + * Uses a ring buffer based rendering task queue with a fixed number of + * rendering slots. The queue is consumed by a dedicated thread + * (rb-render-thread) to dispatch rendering to the GL thread. * * --- * @@ -77,10 +77,6 @@ G_BEGIN_DECLS #define NUM_RENDER_SLOTS 2 #endif -#ifndef PUSH_BUFFER_ENABLED -#define PUSH_BUFFER_ENABLED 1 -#endif - /** * Callback function pointer type for triggering a dynamic fps change. */ @@ -104,7 +100,7 @@ typedef enum { * Slot is currently being rendered. */ RB_BUSY -} RQSlotState; +} RBSlotState; /** * Result status of queuing a buffer for rendering. @@ -133,7 +129,7 @@ typedef enum { */ typedef struct { - // not re-assigned + // not re-assigned, needed as context to dispatch rendering // -------------------------------------------------------------- /** @@ -183,7 +179,7 @@ typedef struct { /** * Usage state of this slot. */ - RQSlotState state; + RBSlotState state; } RBSlot; @@ -210,8 +206,14 @@ typedef struct { */ GstPad *src_pad; + /** + * Utility to properly release GL buffers. + */ BDBufferDisposal buffer_disposal; + /** + * Utility for scheduling downstream push of rendered buffers. + */ PBPushBuffer push_buffer; /** From 171887e247d921d39e8cb50c0daa7c68a49047a0 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sun, 19 Oct 2025 19:49:43 -0500 Subject: [PATCH 26/32] final formatting and nit fixes --- src/bufferdisposal.c | 9 ++++----- src/bufferdisposal.h | 2 +- src/gstglbaseaudiovisualizer.c | 6 +++--- src/gstpmaudiovisualizer.c | 17 +++++++++-------- src/gstprojectmbase.c | 33 +++++++++++++++++---------------- src/pushbuffer.h | 2 +- src/renderbuffer.c | 12 ++++++------ 7 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/bufferdisposal.c b/src/bufferdisposal.c index 716106f..f366467 100644 --- a/src/bufferdisposal.c +++ b/src/bufferdisposal.c @@ -42,7 +42,7 @@ void bd_dispose_gl_buffer(BDBufferDisposal *state, GstBuffer *buf) { } /** - * Used to dispose of dropped gl buffers that are not making it to the src pad. + * Disposal loop for dropped gl buffers that are not making it to the src pad. * Consume buffers to clean-up and dispatch release through gl thread. * * @param user_data Queue state to use. @@ -71,11 +71,10 @@ static gpointer _bd_dispose_thread_func(gpointer user_data) { } /** - * Dispose of all buffered currently queued. + * Dispose of all buffers currently queued. * Needs to be called from the GL thread. * * @param user_data Queue state to use. - * @return NULL */ void bd_clear_queue_gl(GstGLContext *context, gpointer user_data) { BDBufferDisposal *state = (BDBufferDisposal *)user_data; @@ -109,7 +108,6 @@ void bd_init_buffer_disposal(BDBufferDisposal *state, "projectM visualizer plugin buffer cleanup"); } - // init clean up queue state->disposal_thread = NULL; state->gl_context = gl_context; g_atomic_int_set(&state->running, FALSE); @@ -119,6 +117,7 @@ void bd_init_buffer_disposal(BDBufferDisposal *state, void bd_dispose_buffer_disposal(BDBufferDisposal *state) { g_assert(state != NULL); g_assert(state->disposal_thread == NULL); + g_async_queue_unref(state->disposal_queue); state->disposal_queue = NULL; state->gl_context = NULL; @@ -146,4 +145,4 @@ void bd_stop_buffer_disposal(BDBufferDisposal *state) { g_thread_join(state->disposal_thread); state->disposal_thread = NULL; } -} \ No newline at end of file +} diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h index 5dd0f48..9cbb86e 100644 --- a/src/bufferdisposal.h +++ b/src/bufferdisposal.h @@ -86,4 +86,4 @@ void bd_start_buffer_disposal(BDBufferDisposal *state); */ void bd_stop_buffer_disposal(BDBufferDisposal *state); -#endif \ No newline at end of file +#endif diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 564076b..dd33b70 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -72,15 +72,15 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); #define DEFAULT_TIMESTAMP_OFFSET 0 /** - * Allow 0.75 * fps frame duration as wait time for frame render queuing before + * Allow 0.625 * fps frame duration as wait time for frame render queuing before * dropping previous frame. */ #ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 2 +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 5 #endif #ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 3 +#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 8 #endif /* diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 8ce8f4a..2ddcf9f 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -28,9 +28,6 @@ * audio-rate to video-rate and handles renegotiation (downstream video size * changes). * - * It also provides several background shading effects. These effects are - * applied to a previous picture before the `render()` implementation can draw a - * new frame. */ /* @@ -52,7 +49,7 @@ * - Uses a sample count based approach for pts/dts timestamps instead * GstAdapter derived timestamps. * - * - Consistent locking and fixes for some race conditions. + * - Consistent locking, memory (de)allocation * * - Allow dynamic fps adjustments while staying sample accurate. * @@ -834,7 +831,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GST_WARNING_OBJECT(scope, "Segment format not TIME, skipping QoS checks"); } else if (GST_CLOCK_TIME_IS_VALID(earliest_time) && - qostime <= earliest_time) { + qostime < earliest_time) { GstClockTime stream_time, jitter; GstMessage *qos_msg; @@ -847,9 +844,13 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, stream_time = gst_segment_to_stream_time(&scope->priv->segment, GST_FORMAT_TIME, ts); jitter = GST_CLOCK_DIFF(qostime, earliest_time); - qos_msg = - gst_message_new_qos(GST_OBJECT(scope), FALSE, qostime, stream_time, - ts, GST_BUFFER_DURATION(buffer)); + + GstClockTime duration = GST_BUFFER_DURATION_IS_VALID(buffer) + ? GST_BUFFER_DURATION(buffer) + : frame_duration; + + qos_msg = gst_message_new_qos(GST_OBJECT(scope), FALSE, qostime, + stream_time, ts, duration); gst_message_set_qos_values(qos_msg, jitter, proportion, 1000000); gst_message_set_qos_stats(qos_msg, GST_FORMAT_BUFFERS, scope->priv->processed, scope->priv->dropped); diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index bd33fde..c3c5bdd 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -164,7 +164,7 @@ projectm_init(GObject *plugin, GstBaseProjectMSettings *settings, "hard-cut-sensitivity=%f, " "soft-cut-duration=%f, " "preset-duration=%f, " - "mesh-size=(%lu, %lu)" + "mesh-size=(%lu, %lu), " "aspect-correction=%d, " "easter-egg=%f, " "preset-locked=%d, " @@ -282,17 +282,13 @@ void gst_projectm_base_set_property(GObject *object, break; case PROP_MESH_SIZE: { const gchar *meshSizeStr = g_value_get_string(value); - gint width, height; - - gchar **parts = g_strsplit(meshSizeStr, ",", 2); - - if (parts && g_strv_length(parts) == 2) { - width = atoi(parts[0]); - height = atoi(parts[1]); - - settings->mesh_width = width; - settings->mesh_height = height; + if (meshSizeStr) { + gchar **parts = g_strsplit(meshSizeStr, ",", 2); + if (parts[0] && parts[1]) { + settings->mesh_width = atoi(parts[0]); + settings->mesh_height = atoi(parts[1]); + } g_strfreev(parts); } } break; @@ -685,15 +681,18 @@ void gst_projectm_base_install_properties(GObjectClass *gobject_class) { g_param_spec_boolean( "enable-playlist", "Enable Playlist", "Enables or disables the playlist feature. When enabled, the " - "visualizer can switch between presets based on a provided playlist.", + "visualizer can switch between presets based on a provided " + "playlist.", DEFAULT_ENABLE_PLAYLIST, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property( gobject_class, PROP_SHUFFLE_PRESETS, g_param_spec_boolean( "shuffle-presets", "Shuffle Presets", - "Enables or disables preset shuffling. When enabled, the visualizer " - "randomly selects presets from the playlist if presets are provided " + "Enables or disables preset shuffling. When enabled, the " + "visualizer " + "randomly selects presets from the playlist if presets are " + "provided " "and not locked. Playlist must be enabled for this to take effect.", DEFAULT_SHUFFLE_PRESETS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); @@ -711,10 +710,12 @@ void gst_projectm_base_install_properties(GObjectClass *gobject_class) { gobject_class, PROP_IS_LIVE, g_param_spec_string( "is-live", "is live", - "Specifies if the plugin renders in real-time or as fast as possible " + "Specifies if the plugin renders in real-time or as fast as " + "possible " "(offline). This setting is auto-detected for live pipelines, " "but can also be specified if auto-detection is " - "not appropriate. Possible values are \"auto\", \"true\", \"false\". " + "not appropriate. Possible values are \"auto\", \"true\", " + "\"false\". " "Default is \"auto\".", DEFAULT_IS_LIVE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } diff --git a/src/pushbuffer.h b/src/pushbuffer.h index 3d2ea54..6bf0c73 100644 --- a/src/pushbuffer.h +++ b/src/pushbuffer.h @@ -203,4 +203,4 @@ void pb_start_push_buffer(PBPushBuffer *state); */ void pb_stop_push_buffer(PBPushBuffer *state); -#endif \ No newline at end of file +#endif diff --git a/src/renderbuffer.c b/src/renderbuffer.c index 7a49911..a45769d 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -201,7 +201,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, "projectM visualizer plugin render buffer"); } - // context config without ownership + // context without ownership state->plugin = plugin; state->adjust_fps_func = adjust_fps_func; state->gl_context = gl_context; @@ -216,7 +216,7 @@ void rb_init_render_buffer(RBRenderBuffer *state, GstObject *plugin, // changed all the time g_atomic_int_set(&state->running, FALSE); - // init render queue + // init render queue / changed all the time state->render_thread = NULL; state->render_write_idx = NUM_RENDER_SLOTS - 1; state->render_read_idx = NUM_RENDER_SLOTS - 1; @@ -701,14 +701,14 @@ static gpointer _rb_render_thread_func(gpointer user_data) { g_mutex_unlock(&state->slot_lock); // perform gl rendering - const GstClockTime render_time = rb_render_slot(state, slot); + const GstClockTime render_duration = rb_render_slot(state, slot); - // copy params to locals vars to release the slot + // copy params to locals vars before releasing the slot GstBuffer *audio_buffer = slot->in_audio; const GstClockTime frame_duration = slot->frame_duration; const GstClockTime pts = slot->pts; - // copy results to locals vars to release the slot + // copy results to locals vars before releasing the slot GstBuffer *outbuf = slot->out_buf; const gboolean gl_result = slot->gl_result; @@ -723,7 +723,7 @@ static gpointer _rb_render_thread_func(gpointer user_data) { // process rendering fps QoS in case frame was pushed if (state->qos_enabled) { - rb_handle_adaptive_fps_ema(state, render_time, frame_duration); + rb_handle_adaptive_fps_ema(state, render_duration, frame_duration); } } outbuf = NULL; From e89e56f65dd35fd433408b819cbc05d93e69d64f Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Sun, 19 Oct 2025 20:11:07 -0500 Subject: [PATCH 27/32] pad all public structs --- src/bufferdisposal.h | 2 ++ src/gstglbaseaudiovisualizer.h | 7 +++++-- src/gstpmaudiovisualizer.h | 6 ++++++ src/gstprojectm.h | 6 ++++++ src/gstprojectmbase.h | 9 +++++++++ src/pushbuffer.h | 2 ++ src/renderbuffer.h | 6 ++++++ 7 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h index 9cbb86e..a1bba50 100644 --- a/src/bufferdisposal.h +++ b/src/bufferdisposal.h @@ -39,6 +39,8 @@ typedef struct { */ gboolean running; + /*< private >*/ + gpointer _padding[GST_PADDING]; } BDBufferDisposal; /** diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index 090f833..f556f54 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -113,10 +113,10 @@ struct _GstGLBaseAudioVisualizer { */ GstGLBaseAudioVisualizerMode is_live; + GstGLBaseAudioVisualizerPrivate *priv; + /*< private >*/ gpointer _padding[GST_PADDING]; - - GstGLBaseAudioVisualizerPrivate *priv; }; /** @@ -194,6 +194,9 @@ struct _GstAVRenderParams { * Current buffer presentation timestamp. */ GstClockTime pts; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; G_END_DECLS diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index c4544b0..7d31794 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -94,6 +94,9 @@ struct _GstPMAudioVisualizer { * Current pipeline latency. */ GstClockTime latency; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; /** @@ -143,6 +146,9 @@ struct _GstPMAudioVisualizerClass { * Virtual function allow implementor to receive segment change events. */ void (*segment_change)(GstPMAudioVisualizer *scope, GstSegment *segment); + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; GType gst_pm_audio_visualizer_get_type(void); diff --git a/src/gstprojectm.h b/src/gstprojectm.h index bf9239e..2f9d6e0 100644 --- a/src/gstprojectm.h +++ b/src/gstprojectm.h @@ -26,10 +26,16 @@ struct _GstProjectM { GstBaseProjectMSettings settings; GstProjectMPrivate *priv; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; struct _GstProjectMClass { GstGLBaseAudioVisualizerClass parent_class; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; static void gst_projectm_set_property(GObject *object, guint prop_id, diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h index e7107e0..6f458c5 100644 --- a/src/gstprojectmbase.h +++ b/src/gstprojectmbase.h @@ -37,6 +37,9 @@ struct _GstBaseProjectMSettings { gboolean shuffle_presets; gint min_fps_n; gint min_fps_d; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; /** @@ -49,6 +52,9 @@ struct _GstBaseProjectMPrivate { GstClockTime first_frame_time; gboolean first_frame_received; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; /** @@ -58,6 +64,9 @@ struct _GstBaseProjectMInitResult { projectm_handle ret_handle; projectm_playlist_handle ret_playlist; gboolean success; + + /*< private >*/ + gpointer _padding[GST_PADDING]; }; typedef struct _GstBaseProjectMPrivate GstBaseProjectMPrivate; diff --git a/src/pushbuffer.h b/src/pushbuffer.h index 6bf0c73..59a9ae8 100644 --- a/src/pushbuffer.h +++ b/src/pushbuffer.h @@ -120,6 +120,8 @@ typedef struct { */ gboolean avg_jitter_init; + /*< private >*/ + gpointer _padding[GST_PADDING]; } PBPushBuffer; /** diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 61764b1..2949505 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -181,6 +181,8 @@ typedef struct { */ RBSlotState state; + /*< private >*/ + gpointer _padding[GST_PADDING]; } RBSlot; /** @@ -302,6 +304,8 @@ typedef struct { */ guint64 ema_smoothed_render_time; + /*< private >*/ + gpointer _padding[GST_PADDING]; } RBRenderBuffer; /** @@ -334,6 +338,8 @@ typedef struct { */ GstBuffer *in_audio; + /*< private >*/ + gpointer _padding[GST_PADDING]; } RBQueueArgs; /** From 2c23e63ecddaa8dfd5683f717dc56fa120dae8a3 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 20 Oct 2025 01:09:06 -0500 Subject: [PATCH 28/32] fix include ordering --- src/bufferdisposal.c | 3 +++ src/debug.c | 4 ++-- src/gstglbaseaudiovisualizer.c | 13 +++++++------ src/gstpmaudiovisualizer.c | 4 ++-- src/gstprojectm.c | 4 ++-- src/gstprojectmbase.c | 5 +++-- src/gstprojectmbase.h | 1 + src/gstprojectmcaps.c | 7 ++++--- src/pushbuffer.c | 7 +++++-- src/register.c | 4 ++-- src/renderbuffer.c | 4 ++-- 11 files changed, 33 insertions(+), 23 deletions(-) diff --git a/src/bufferdisposal.c b/src/bufferdisposal.c index f366467..3a40335 100644 --- a/src/bufferdisposal.c +++ b/src/bufferdisposal.c @@ -1,3 +1,6 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif #include "bufferdisposal.h" diff --git a/src/debug.c b/src/debug.c index f269b7e..510b480 100644 --- a/src/debug.c +++ b/src/debug.c @@ -2,11 +2,11 @@ #include "config.h" #endif +#include "debug.h" + #include #include -#include "debug.h" - void gl_error_handler(GstGLContext *context) { GLuint error = context->gl_vtable->GetError(); diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index dd33b70..f20e1c7 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -34,12 +34,13 @@ #include "config.h" #endif -#include - #include "gstglbaseaudiovisualizer.h" + #include "gstpmaudiovisualizer.h" #include "renderbuffer.h" +#include + /** * SECTION:GstGLBaseAudioVisualizer * @short_description: #GstPMAudioVisualizer subclass for injecting OpenGL @@ -72,8 +73,8 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); #define DEFAULT_TIMESTAMP_OFFSET 0 /** - * Allow 0.625 * fps frame duration as wait time for frame render queuing before - * dropping previous frame. + * Wait for up to 0.625 * fps frame duration for the previous frame to start + * rendering, otherwise the previous frame is dropped. */ #ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N #define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 5 @@ -147,12 +148,12 @@ gst_gl_base_audio_visualizer_parent_render(GstPMAudioVisualizer *bscope, guint64 frame_duration); /** - * Internal utility for resetting state on start \ + * Internal utility for resetting state on start. */ static void gst_gl_base_audio_visualizer_start(GstGLBaseAudioVisualizer *glav); /** - * Internal utility for cleaning up gl context on stop + * Internal utility for cleaning up gl context on stop. */ static void gst_gl_base_audio_visualizer_stop(GstGLBaseAudioVisualizer *glav); diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 2ddcf9f..2e5f49a 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -67,13 +67,13 @@ #include "config.h" #endif +#include "gstpmaudiovisualizer.h" + #include #include #include -#include "gstpmaudiovisualizer.h" - GST_DEBUG_CATEGORY_STATIC(pm_audio_visualizer_debug); #define GST_CAT_DEFAULT (pm_audio_visualizer_debug) diff --git a/src/gstprojectm.c b/src/gstprojectm.c index 858f8b3..853aaae 100644 --- a/src/gstprojectm.c +++ b/src/gstprojectm.c @@ -6,14 +6,14 @@ #include #endif -#include - #include "gstprojectm.h" #include "debug.h" #include "gstglbaseaudiovisualizer.h" #include "gstprojectmcaps.h" +#include + GST_DEBUG_CATEGORY_STATIC(gst_projectm_debug); #define GST_CAT_DEFAULT gst_projectm_debug diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index c3c5bdd..0e7cd19 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -3,13 +3,14 @@ #include "config.h" #endif -#include +#include "gstprojectmbase.h" #include "debug.h" #include "gstglbaseaudiovisualizer.h" -#include "gstprojectmbase.h" #include "gstprojectmconfig.h" +#include + GST_DEBUG_CATEGORY_STATIC(gst_projectm_base_debug); #define GST_CAT_DEFAULT gst_projectm_base_debug diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h index 6f458c5..63bf2cd 100644 --- a/src/gstprojectmbase.h +++ b/src/gstprojectmbase.h @@ -7,6 +7,7 @@ #ifndef __GST_PROJECTM_BASE_H__ #define __GST_PROJECTM_BASE_H__ +#include #include #include #include diff --git a/src/gstprojectmcaps.c b/src/gstprojectmcaps.c index e146420..e022c8b 100644 --- a/src/gstprojectmcaps.c +++ b/src/gstprojectmcaps.c @@ -3,11 +3,12 @@ #include #endif -#include -#include +#include "gstprojectmcaps.h" #include "gstprojectm.h" -#include "gstprojectmcaps.h" + +#include +#include GST_DEBUG_CATEGORY_STATIC(gst_projectm_caps_debug); #define GST_CAT_DEFAULT gst_projectm_caps_debug diff --git a/src/pushbuffer.c b/src/pushbuffer.c index 1cec410..e977ee8 100644 --- a/src/pushbuffer.c +++ b/src/pushbuffer.c @@ -1,10 +1,13 @@ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif #include "pushbuffer.h" -#include - #include "bufferdisposal.h" +#include + GST_DEBUG_CATEGORY_STATIC(pushbuffer_debug); #define GST_CAT_DEFAULT pushbuffer_debug diff --git a/src/register.c b/src/register.c index afe3b5d..1ed0a03 100644 --- a/src/register.c +++ b/src/register.c @@ -3,11 +3,11 @@ #include "config.h" #endif -#include - #include "gstprojectm.h" #include "gstprojectmconfig.h" +#include + /* * This unit registers all gst elements from this plugin library to make them * available to GStreamer. diff --git a/src/renderbuffer.c b/src/renderbuffer.c index a45769d..b2bcad4 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -3,10 +3,10 @@ #include "config.h" #endif -#include - #include "renderbuffer.h" +#include + GST_DEBUG_CATEGORY_STATIC(renderbuffer_debug); #define GST_CAT_DEFAULT renderbuffer_debug From 362e4a0e2dcad52aa7f73519c8794dd698bda9e0 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Mon, 20 Oct 2025 01:49:48 -0500 Subject: [PATCH 29/32] release buffer pool on dispose --- src/gstpmaudiovisualizer.c | 46 ++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 2e5f49a..24db605 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -360,6 +360,12 @@ static void gst_pm_audio_visualizer_get_property(GObject *object, guint prop_id, static void gst_pm_audio_visualizer_dispose(GObject *object) { GstPMAudioVisualizer *scope = GST_PM_AUDIO_VISUALIZER(object); + // make sure nobody is still waiting to get started + g_mutex_lock(&scope->priv->config_lock); + scope->priv->ready = TRUE; + g_cond_broadcast(&scope->priv->ready_cond); + g_mutex_unlock(&scope->priv->config_lock); + if (scope->priv->adapter) { g_object_unref(scope->priv->adapter); scope->priv->adapter = NULL; @@ -368,6 +374,25 @@ static void gst_pm_audio_visualizer_dispose(GObject *object) { gst_buffer_unref(scope->priv->inbuf); scope->priv->inbuf = NULL; } + + GST_OBJECT_LOCK(scope); + if (scope->priv->pool) { + if (scope->priv->pool_active) + gst_buffer_pool_set_active(scope->priv->pool, FALSE); + gst_object_unref(scope->priv->pool); + scope->priv->pool = NULL; + scope->priv->pool_active = FALSE; + } + if (scope->priv->allocator) { + gst_object_unref(scope->priv->allocator); + scope->priv->allocator = NULL; + } + if (scope->priv->query) { + gst_query_unref(scope->priv->query); + scope->priv->query = NULL; + } + GST_OBJECT_UNLOCK(scope); + g_mutex_clear(&scope->priv->config_lock); g_cond_clear(&scope->priv->ready_cond); @@ -746,14 +771,21 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, scope->priv->pts_offset_initialized = TRUE; scope->priv->pts_offset = GST_BUFFER_PTS(buffer); - GstClock *clock = gst_element_get_clock(GST_ELEMENT(scope)); - GstClockTime running_time = gst_clock_get_time(clock) - - gst_element_get_base_time(GST_ELEMENT(scope)); + if (gst_debug_category_get_threshold(pm_audio_visualizer_debug) >= + GST_LEVEL_INFO) { + GstClock *clock = gst_element_get_clock(GST_ELEMENT(scope)); + GstClockTime running_time = 0; + if (clock != NULL) { + running_time = gst_clock_get_time(clock) - + gst_element_get_base_time(GST_ELEMENT(scope)); + gst_object_unref(clock); + } - GST_DEBUG_OBJECT( - scope, - "Buffer ts: %" GST_TIME_FORMAT ", running_time: %" GST_TIME_FORMAT, - GST_TIME_ARGS(scope->priv->pts_offset), GST_TIME_ARGS(running_time)); + GST_DEBUG_OBJECT( + scope, + "Buffer ts: %" GST_TIME_FORMAT ", running_time: %" GST_TIME_FORMAT, + GST_TIME_ARGS(scope->priv->pts_offset), GST_TIME_ARGS(running_time)); + } } g_mutex_unlock(&scope->priv->config_lock); From 3a84a6a0555374f20706b789fdaa602d0f561160 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Wed, 22 Oct 2025 01:01:54 -0500 Subject: [PATCH 30/32] windows build fixes --- CMakeLists.txt | 2 +- src/bufferdisposal.c | 2 ++ src/bufferdisposal.h | 2 +- src/gstglbaseaudiovisualizer.c | 7 ++++++- src/gstglbaseaudiovisualizer.h | 5 ++--- src/gstpmaudiovisualizer.c | 2 +- src/gstprojectm.c | 3 --- src/gstprojectm.h | 2 -- src/gstprojectmbase.c | 36 ++++++++++++++++++---------------- src/gstprojectmbase.h | 9 ++------- src/register.c | 26 +++++++++++++++++------- src/renderbuffer.c | 2 ++ src/renderbuffer.h | 7 ++++--- 13 files changed, 59 insertions(+), 46 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 06f8850..726a51f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,7 @@ find_package(projectM4 4.1.0 REQUIRED Playlist) find_package(GStreamer REQUIRED COMPONENTS gstreamer-audio gstreamer-gl gstreamer-pbutils gstreamer-video) find_package(GLIB2 REQUIRED) -add_library(gstprojectm SHARED +add_library(gstprojectm MODULE src/bufferdisposal.h src/bufferdisposal.c src/debug.h diff --git a/src/bufferdisposal.c b/src/bufferdisposal.c index 3a40335..1b86988 100644 --- a/src/bufferdisposal.c +++ b/src/bufferdisposal.c @@ -4,6 +4,8 @@ #include "bufferdisposal.h" +#include + GST_DEBUG_CATEGORY_STATIC(buffercleanup_debug); #define GST_CAT_DEFAULT buffercleanup_debug diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h index a1bba50..06ce6a5 100644 --- a/src/bufferdisposal.h +++ b/src/bufferdisposal.h @@ -7,8 +7,8 @@ #ifndef __BUFFERDISPOSAL_H__ #define __BUFFERDISPOSAL_H__ -#include #include +#include typedef struct { diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index f20e1c7..ac50893 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -39,8 +39,13 @@ #include "gstpmaudiovisualizer.h" #include "renderbuffer.h" +#include #include +#ifdef _WIN32 +#define strcasecmp _stricmp +#endif + /** * SECTION:GstGLBaseAudioVisualizer * @short_description: #GstPMAudioVisualizer subclass for injecting OpenGL @@ -432,7 +437,7 @@ static gboolean is_pipeline_live(GstElement *element) { if (parent && GST_IS_PIPELINE(parent)) { pipeline = GST_PIPELINE(parent); - is_live = gst_pipeline_is_live(pipeline); + is_live = TRUE; // gst_pipeline_is_live(pipeline); gst_object_unref(parent); } diff --git a/src/gstglbaseaudiovisualizer.h b/src/gstglbaseaudiovisualizer.h index f556f54..52ba6d2 100644 --- a/src/gstglbaseaudiovisualizer.h +++ b/src/gstglbaseaudiovisualizer.h @@ -32,10 +32,10 @@ #ifndef __GST_GL_BASE_AUDIO_VISUALIZER_H__ #define __GST_GL_BASE_AUDIO_VISUALIZER_H__ -#include - #include "gstpmaudiovisualizer.h" +#include + typedef struct _GstGLBaseAudioVisualizer GstGLBaseAudioVisualizer; typedef struct _GstGLBaseAudioVisualizerClass GstGLBaseAudioVisualizerClass; typedef struct _GstGLBaseAudioVisualizerPrivate GstGLBaseAudioVisualizerPrivate; @@ -45,7 +45,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstGLBaseAudioVisualizer, gst_object_unref) G_BEGIN_DECLS -GST_GL_API GType gst_gl_base_audio_visualizer_get_type(void); #define GST_TYPE_GL_BASE_AUDIO_VISUALIZER \ diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 24db605..703a35b 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -863,7 +863,7 @@ static GstFlowReturn gst_pm_audio_visualizer_chain(GstPad *pad, GST_WARNING_OBJECT(scope, "Segment format not TIME, skipping QoS checks"); } else if (GST_CLOCK_TIME_IS_VALID(earliest_time) && - qostime < earliest_time) { + qostime <= earliest_time) { GstClockTime stream_time, jitter; GstMessage *qos_msg; diff --git a/src/gstprojectm.c b/src/gstprojectm.c index 853aaae..557bfe0 100644 --- a/src/gstprojectm.c +++ b/src/gstprojectm.c @@ -2,9 +2,6 @@ #include "config.h" #endif -#ifdef USE_GLEW -#include -#endif #include "gstprojectm.h" diff --git a/src/gstprojectm.h b/src/gstprojectm.h index 2f9d6e0..e16d33a 100644 --- a/src/gstprojectm.h +++ b/src/gstprojectm.h @@ -56,8 +56,6 @@ static gboolean gst_projectm_fill_gl_memory(GstAVRenderParams *render_data); static void gst_projectm_class_init(GstProjectMClass *klass); -static gboolean plugin_init(GstPlugin *plugin); - static gboolean gst_projectm_setup(GstGLBaseAudioVisualizer *glav); G_END_DECLS diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index 0e7cd19..f8062e9 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -9,7 +9,11 @@ #include "gstglbaseaudiovisualizer.h" #include "gstprojectmconfig.h" -#include +#include + +#ifdef USE_GLEW +#include +#endif GST_DEBUG_CATEGORY_STATIC(gst_projectm_base_debug); #define GST_CAT_DEFAULT gst_projectm_base_debug @@ -57,11 +61,6 @@ enum { #define DEFAULT_MIN_FPS_D 1 #define DEFAULT_IS_LIVE "auto" -void gst_projectm_base_init_once() { - GST_DEBUG_CATEGORY_INIT(gst_projectm_base_debug, "projectm_base", 0, - "projectM visualizer plugin base"); -} - static gboolean gst_projectm_base_log_preset_change(gpointer preset) { GST_INFO("Preset: %s", (char *)preset); @@ -408,6 +407,13 @@ void gst_projectm_base_get_property(GObject *object, void gst_projectm_base_init(GstBaseProjectMSettings *settings, GstBaseProjectMPrivate *priv) { + static gsize _debug_initialized = 0; + if (g_once_init_enter(&_debug_initialized)) + { + GST_DEBUG_CATEGORY_INIT(gst_projectm_base_debug, "projectm_base", 0, + "projectM visualizer plugin base"); + } + // Set default values for properties settings->preset_path = DEFAULT_PRESET_PATH; settings->texture_dir_path = DEFAULT_TEXTURE_DIR_PATH; @@ -421,20 +427,16 @@ void gst_projectm_base_init(GstBaseProjectMSettings *settings, settings->shuffle_presets = DEFAULT_SHUFFLE_PRESETS; settings->min_fps_d = DEFAULT_MIN_FPS_D; settings->min_fps_n = DEFAULT_MIN_FPS_N; - settings->is_live = strdup(DEFAULT_IS_LIVE); + settings->is_live = g_strdup(DEFAULT_IS_LIVE); const gchar *meshSizeStr = DEFAULT_MESH_SIZE; - gint width, height; - - gchar **parts = g_strsplit(meshSizeStr, ",", 2); - - if (parts && g_strv_length(parts) == 2) { - width = atoi(parts[0]); - height = atoi(parts[1]); - - settings->mesh_width = width; - settings->mesh_height = height; + if (meshSizeStr) { + gchar **parts = g_strsplit(meshSizeStr, ",", 2); + if (parts[0] && parts[1]) { + settings->mesh_width = atoi(parts[0]); + settings->mesh_height = atoi(parts[1]); + } g_strfreev(parts); } diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h index 63bf2cd..7ba47c0 100644 --- a/src/gstprojectmbase.h +++ b/src/gstprojectmbase.h @@ -7,8 +7,9 @@ #ifndef __GST_PROJECTM_BASE_H__ #define __GST_PROJECTM_BASE_H__ -#include #include +#include +#include #include #include @@ -74,12 +75,6 @@ typedef struct _GstBaseProjectMPrivate GstBaseProjectMPrivate; typedef struct _GstBaseProjectMSettings GstBaseProjectMSettings; typedef struct _GstBaseProjectMInitResult GstBaseProjectMInitResult; -/** - * One time initialization. Should be called once before any other function in - * this unit. - */ -void gst_projectm_base_init_once(); - /** * get_property delegate for projectM setting structs. * diff --git a/src/register.c b/src/register.c index 1ed0a03..2267010 100644 --- a/src/register.c +++ b/src/register.c @@ -3,18 +3,23 @@ #include "config.h" #endif +#ifdef _WIN32 +#define EXPORT __declspec(dllexport) +#else +#define EXPORT +#endif + #include "gstprojectm.h" #include "gstprojectmconfig.h" #include + /* * This unit registers all gst elements from this plugin library to make them * available to GStreamer. */ -static gboolean plugin_init(GstPlugin *plugin) { - - gst_projectm_base_init_once(); +EXPORT gboolean plugin_init(GstPlugin *plugin) { // register main plugin projectM element gboolean p1 = gst_element_register(plugin, "projectm", GST_RANK_NONE, @@ -25,7 +30,14 @@ static gboolean plugin_init(GstPlugin *plugin) { return p1; } -GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, projectm, - "plugin to visualize audio using the ProjectM library", - plugin_init, PACKAGE_VERSION, PACKAGE_LICENSE, PACKAGE_NAME, - PACKAGE_ORIGIN) +GST_PLUGIN_DEFINE ( + GST_VERSION_MAJOR, + GST_VERSION_MINOR, + projectm, + "plugin to visualize audio using the ProjectM library", + plugin_init, + PACKAGE_VERSION, + PACKAGE_LICENSE, + PACKAGE_NAME, + PACKAGE_ORIGIN +) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index b2bcad4..afb5419 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -7,6 +7,8 @@ #include +#include + GST_DEBUG_CATEGORY_STATIC(renderbuffer_debug); #define GST_CAT_DEFAULT renderbuffer_debug diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 2949505..476c0c7 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -52,12 +52,13 @@ #ifndef __RENDERBUFFER_H__ #define __RENDERBUFFER_H__ -#include -#include - #include "bufferdisposal.h" #include "pushbuffer.h" +#include +#include +#include + G_BEGIN_DECLS /** From 2700c8b77eedbfb1f7158a94ceece9c591f7351b Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Wed, 22 Oct 2025 23:54:01 -0500 Subject: [PATCH 31/32] windows build fix, fix lock congestion, update scripts --- README.md | 7 +++--- build.ps1 | 4 ++-- build.sh | 4 +--- convert.sh | 1 + docs/LINUX.md | 2 +- docs/OSX.md | 2 +- docs/OVERVIEW.md | 2 +- docs/WINDOWS.md | 2 +- src/bufferdisposal.h | 2 +- src/gstglbaseaudiovisualizer.c | 32 ++++++++++++++++--------- src/gstprojectm.c | 1 - src/gstprojectmbase.c | 5 ++-- src/gstprojectmbase.h | 2 +- src/register.c | 16 ++++--------- src/renderbuffer.c | 44 +++++++++++++++++++--------------- src/renderbuffer.h | 14 ++++++++--- test.ps1 | 13 ++++++---- test.sh | 15 ++++++------ 18 files changed, 93 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index acb5951..8a8abe1 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ The documentation has been organized into distinct files, each dedicated to a sp Once the plugin has been installed, you can use it something like this to render to an OpenGL window: ```shell -gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=10 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false +gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=10 mesh-size=48,32 is-live=true ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false ``` To render from a live source in real-time to a gl window, an identity element can be used to provide a proper timestamp source for the pipeline. This example also includes a texture directory: @@ -267,8 +267,9 @@ A **fixed number of audio samples is consumed per video frame**. **Example:** `735 samples per frame at 44.1 kHz = ~60 FPS.` -**Note:** Live pipelines are auto-detected by the plugin. For cases where auto-detection is not appropriate, -the `is-live` property can be configured. +**Note:** Live pipelines are auto-detected by the plugin if Gstreamer supports it (not supported on Windows). +For Windows or other cases where auto-detection is not appropriate, the `is-live` property can be configured. +The default mode is offline rendering, `is-live=false`. **Live pipelines only:** Frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with pipeline caps FPS. diff --git a/build.ps1 b/build.ps1 index cf25cae..8aa1db9 100644 --- a/build.ps1 +++ b/build.ps1 @@ -54,7 +54,7 @@ function Start-ConfigureBuild { -DVCPKG_TARGET_TRIPLET=x64-windows ` -DCMAKE_MSVC_RUNTIME_LIBRARY="MultiThreaded$<$:Debug>DLL" ` -DCMAKE_VERBOSE_MAKEFILE=YES ` - -DCMAKE_PREFIX_PATH="${Env:PROJECTM_ROOT}/lib/cmake/projectM4" + -DCMAKE_PREFIX_PATH="${Env:PROJECTM_ROOT}" } # Copy required DLLs to dist directory @@ -137,7 +137,7 @@ function Invoke-PromptInstall { # Print example command Write-Host Write-Host "Done! Here's an example command:" - Write-Host 'gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false' + Write-Host 'gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false' } else { Write-Host diff --git a/build.sh b/build.sh index 74a1148..940d8f0 100755 --- a/build.sh +++ b/build.sh @@ -4,10 +4,8 @@ set -e # Set variables based on OS if [[ "$OSTYPE" == "linux-gnu"* ]]; then LIB_EXT="so" - VIDEO_SINK="xvimagesink" elif [[ "$OSTYPE" == "darwin"* ]]; then LIB_EXT="dylib" - VIDEO_SINK="osxvideosink" else echo "Unsupported OS!" exit 1 @@ -99,7 +97,7 @@ prompt_install() { # Print example command echo echo "Done! Here's an example command:" - echo "gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! $VIDEO_SINK sync=false" + echo "gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! \"video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1\" ! videoconvert ! glimagesink sync=false" else echo echo "Done!" diff --git a/convert.sh b/convert.sh index c709008..9059e99 100644 --- a/convert.sh +++ b/convert.sh @@ -159,6 +159,7 @@ gst-launch-1.0 -e \ preset=$PRESET_PATH \ texture-dir=$TEXTURE_DIR \ preset-duration=$PRESET_DURATION \ + is-live=false \ mesh-size=${MESH_X},${MESH_Y} ! \ identity sync=false ! videoconvert ! videorate ! \ video/x-raw\(memory:GLMemory\),framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \ diff --git a/docs/LINUX.md b/docs/LINUX.md index 6315df6..2443262 100644 --- a/docs/LINUX.md +++ b/docs/LINUX.md @@ -92,7 +92,7 @@ source ~/.bash_profile To utilize the plugin with the example, please install GStreamer ```bash -gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false +gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false ``` ### Testing diff --git a/docs/OSX.md b/docs/OSX.md index bb35f42..429bedc 100644 --- a/docs/OSX.md +++ b/docs/OSX.md @@ -102,7 +102,7 @@ source ~/.bash_profile To utilize the plugin with the example, please install GStreamer ```bash -gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false +gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false ``` ### Testing diff --git a/docs/OVERVIEW.md b/docs/OVERVIEW.md index d6be7ec..24b73ab 100644 --- a/docs/OVERVIEW.md +++ b/docs/OVERVIEW.md @@ -15,7 +15,7 @@ - [x] OSX - [ ] Windows (see issues) - [x] Accepting an audio/x-raw stream (coded to add more formats later, if needed) -- [x] Generating a video/x-raw stream (coded to add more formats later, if needed) +- [x] Generating a video/x-raw(memory:GLMemory) stream (coded to add more formats later, if needed) - [x] Utilizing the new C API in libprojectM 4.0 - [x] Implemented properties with defaults (aka settings) diff --git a/docs/WINDOWS.md b/docs/WINDOWS.md index db4d442..b570757 100644 --- a/docs/WINDOWS.md +++ b/docs/WINDOWS.md @@ -66,7 +66,7 @@ Copy-Item -Path "dist\gstprojectm.dll" -Destination "$Env:USERPROFILE\.gstreamer To utilize the plugin with the example, please install GStreamer ```powershell -gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false +gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false ``` ### Testing diff --git a/src/bufferdisposal.h b/src/bufferdisposal.h index 06ce6a5..9f7db5c 100644 --- a/src/bufferdisposal.h +++ b/src/bufferdisposal.h @@ -7,8 +7,8 @@ #ifndef __BUFFERDISPOSAL_H__ #define __BUFFERDISPOSAL_H__ -#include #include +#include typedef struct { diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index ac50893..37b7c04 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -39,8 +39,8 @@ #include "gstpmaudiovisualizer.h" #include "renderbuffer.h" -#include #include +#include #ifdef _WIN32 #define strcasecmp _stricmp @@ -78,15 +78,16 @@ GST_DEBUG_CATEGORY_STATIC(gst_gl_base_audio_visualizer_debug); #define DEFAULT_TIMESTAMP_OFFSET 0 /** - * Wait for up to 0.625 * fps frame duration for the previous frame to start - * rendering, otherwise the previous frame is dropped. + * Wait for up to 0.625 * fps frame duration for a free slot to queue input + * audio for a frame. If the previous frame does not start rendering within this + * time, it is dropped. */ -#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 5 +#ifndef MAX_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N +#define MAX_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N 5 #endif -#ifndef MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D -#define MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 8 +#ifndef MAX_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D +#define MAX_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D 8 #endif /* @@ -418,11 +419,14 @@ static void gst_gl_base_audio_visualizer_set_context(GstElement *element, /** * Find the pipeline and determine if it is live. + * Not supported on Windows. * * @param element Plugin element. * * @return TRUE if the pipeline is live. */ +#ifndef _WIN32 + static gboolean is_pipeline_live(GstElement *element) { GstPipeline *pipeline = NULL; gboolean is_live = FALSE; @@ -437,13 +441,15 @@ static gboolean is_pipeline_live(GstElement *element) { if (parent && GST_IS_PIPELINE(parent)) { pipeline = GST_PIPELINE(parent); - is_live = TRUE; // gst_pipeline_is_live(pipeline); + is_live = gst_pipeline_is_live(pipeline); gst_object_unref(parent); } return is_live; } +#endif + static gboolean gst_gl_base_audio_visualizer_default_gl_start(GstGLBaseAudioVisualizer *glav) { return TRUE; @@ -484,8 +490,12 @@ static void gst_gl_base_audio_visualizer_gl_start(GstGLContext *context, } else if (glav->is_live == GST_GL_BASE_AUDIO_VISUALIZER_REALTIME) { glav->priv->is_realtime = TRUE; } else { - // auto detect + // auto-detect, unless we're on windows +#ifdef _WIN32 + glav->priv->is_realtime = FALSE; +#else glav->priv->is_realtime = is_pipeline_live(GST_ELEMENT(data)); +#endif } // render loop QoS is disabled for offline rendering @@ -628,8 +638,8 @@ static GstFlowReturn gst_gl_base_audio_visualizer_fill( // limit wait based on fps factor, make sure we never wait too long in order // to keep in sync args.max_wait = (GstClockTimeDiff)gst_util_uint64_scale_int( - frame_duration, MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N, - MAX_RENDER_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D); + frame_duration, MAX_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_N, + MAX_QUEUE_WAIT_TIME_IN_FRAME_DURATIONS_D); g_rec_mutex_unlock(&glav->priv->context_lock); diff --git a/src/gstprojectm.c b/src/gstprojectm.c index 557bfe0..0937539 100644 --- a/src/gstprojectm.c +++ b/src/gstprojectm.c @@ -2,7 +2,6 @@ #include "config.h" #endif - #include "gstprojectm.h" #include "debug.h" diff --git a/src/gstprojectmbase.c b/src/gstprojectmbase.c index f8062e9..59b1e40 100644 --- a/src/gstprojectmbase.c +++ b/src/gstprojectmbase.c @@ -408,10 +408,9 @@ void gst_projectm_base_init(GstBaseProjectMSettings *settings, GstBaseProjectMPrivate *priv) { static gsize _debug_initialized = 0; - if (g_once_init_enter(&_debug_initialized)) - { + if (g_once_init_enter(&_debug_initialized)) { GST_DEBUG_CATEGORY_INIT(gst_projectm_base_debug, "projectm_base", 0, - "projectM visualizer plugin base"); + "projectM visualizer plugin base"); } // Set default values for properties diff --git a/src/gstprojectmbase.h b/src/gstprojectmbase.h index 7ba47c0..bb83522 100644 --- a/src/gstprojectmbase.h +++ b/src/gstprojectmbase.h @@ -7,8 +7,8 @@ #ifndef __GST_PROJECTM_BASE_H__ #define __GST_PROJECTM_BASE_H__ -#include #include +#include #include #include #include diff --git a/src/register.c b/src/register.c index 2267010..342c46d 100644 --- a/src/register.c +++ b/src/register.c @@ -14,7 +14,6 @@ #include - /* * This unit registers all gst elements from this plugin library to make them * available to GStreamer. @@ -30,14 +29,7 @@ EXPORT gboolean plugin_init(GstPlugin *plugin) { return p1; } -GST_PLUGIN_DEFINE ( - GST_VERSION_MAJOR, - GST_VERSION_MINOR, - projectm, - "plugin to visualize audio using the ProjectM library", - plugin_init, - PACKAGE_VERSION, - PACKAGE_LICENSE, - PACKAGE_NAME, - PACKAGE_ORIGIN -) +GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, projectm, + "plugin to visualize audio using the ProjectM library", + plugin_init, PACKAGE_VERSION, PACKAGE_LICENSE, PACKAGE_NAME, + PACKAGE_ORIGIN) diff --git a/src/renderbuffer.c b/src/renderbuffer.c index afb5419..439a3b1 100644 --- a/src/renderbuffer.c +++ b/src/renderbuffer.c @@ -274,20 +274,19 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { gint slot_index = 0; gboolean found_slot = FALSE; - while (!found_slot) { + while (!found_slot && g_atomic_int_get(&state->running)) { // next slot to insert to slot_index = (state->render_write_idx + 1) % NUM_RENDER_SLOTS; - slot = &state->slots[slot_index]; // jump over busy slot that's currently rendering if needed - if (slot->state == RB_BUSY) { + if (state->slots[slot_index].state == RB_BUSY) { slot_index = (state->render_write_idx + 2) % NUM_RENDER_SLOTS; } + slot = &state->slots[slot_index]; // in case there is only one slot, it may still be busy - found_slot = - slot->state != RB_BUSY && (wait_is_limited || slot->state == RB_EMPTY); + found_slot = slot->state == RB_EMPTY; if (!found_slot) { if (wait_is_limited) { @@ -317,6 +316,10 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { } } + if (slot == NULL || !g_atomic_int_get(&state->running)) { + return RB_STOPPED; + } + if (slot->state == RB_BUSY) { // out of time, and we still can't schedule g_mutex_unlock(&state->slot_lock); @@ -327,7 +330,13 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { // evict if already in use and clear buffers const gboolean is_evicted = slot->state == RB_READY; + // optimization: set slot to empty so render thread does not pick this up yet + // this is safe, since we're being called by the chain function only (single + // producer) + slot->state = RB_EMPTY; + g_mutex_unlock(&state->slot_lock); + // do heavy stuff if (slot->in_audio != NULL) { gst_buffer_unref(slot->in_audio); } @@ -338,23 +347,20 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args) { } // populate slot - slot->state = RB_READY; slot->gl_result = FALSE; slot->pts = args->pts; slot->frame_duration = args->frame_duration; slot->in_audio = gst_buffer_copy_deep(args->in_audio); + // lock and mark ready + g_mutex_lock(&state->slot_lock); + slot->state = RB_READY; + // signal render thread that there is something to do g_cond_signal(&state->render_queued_cond); g_mutex_unlock(&state->slot_lock); - RBQueueResult result; - if (found_slot) { - result = is_evicted == FALSE ? RB_SUCCESS : RB_EVICTED; - } else { - result = RB_TIMEOUT; - } - return result; + return is_evicted == FALSE ? RB_SUCCESS : RB_EVICTED; } void rb_queue_render_task_log(RBQueueArgs *args) { @@ -391,7 +397,7 @@ void rb_queue_render_task_log(RBQueueArgs *args) { break; } - case RB_SUCCESS: + default: break; } } @@ -444,7 +450,7 @@ gboolean rb_is_render_too_late(GstElement *element, const GstClockTime latency, * @param state Render buffer to use. * @param slot Prepared slot to render. * - * @return Render duration. + * @return Time it took to render the frame. */ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { // measure rendering for QoS @@ -453,21 +459,21 @@ GstClockTime rb_render_slot(RBRenderBuffer *state, RBSlot *slot) { // Dispatch slot to GL thread gst_gl_context_thread_add(state->gl_context, slot->gl_fill_func, slot); - const GstClockTime render_time = + const GstClockTime render_duration = GST_CLOCK_DIFF(render_start, gst_util_get_timestamp()); // render took longer than the frame duration, this is a problem for // real-time rendering if it happens too often - if (render_time > slot->frame_duration) { + if (render_duration > slot->frame_duration) { GST_DEBUG_OBJECT( state->plugin, "Render GL frame took too long: %" GST_TIME_FORMAT ", frame-duration: %" GST_TIME_FORMAT ", pts: %" GST_TIME_FORMAT, - GST_TIME_ARGS(render_time), GST_TIME_ARGS(slot->frame_duration), + GST_TIME_ARGS(render_duration), GST_TIME_ARGS(slot->frame_duration), GST_TIME_ARGS(slot->pts)); } - return render_time; + return render_duration; } /** diff --git a/src/renderbuffer.h b/src/renderbuffer.h index 476c0c7..194c688 100644 --- a/src/renderbuffer.h +++ b/src/renderbuffer.h @@ -55,9 +55,9 @@ #include "bufferdisposal.h" #include "pushbuffer.h" -#include #include #include +#include G_BEGIN_DECLS @@ -68,7 +68,7 @@ G_BEGIN_DECLS * available for queuing the next audio buffer to render. * * Note: Increasing the number of slots >2 is not fully supported since - * it would require handling of PTS offset changes. See inline code comments. + * it would require handling of PTS offset changes. See comments in code. * * Valid values: * 1 : Wait for previous render to complete before scheduling. @@ -120,7 +120,11 @@ typedef enum { /** * Buffer could not be queued because the allowed wait could not be met. */ - RB_TIMEOUT + RB_TIMEOUT, + /** + * Buffer could not be queued because the buffer was stopped. + */ + RB_STOPPED } RBQueueResult; /** @@ -374,6 +378,8 @@ void rb_dispose_render_buffer(RBRenderBuffer *state); * Queue an audio buffer for rendering. The queuing is guaranteed to return * within the given max time budget. The buffer will be dropped if queuing is * not possible within the given time budget. + * Note: Single producer function. Not thread safe, should be called from chain + * function only! * * @param args Audio buffer and frame details for rendering. The render buffer * does not take ownership of the given pointer. The given audio buffer is @@ -385,6 +391,8 @@ RBQueueResult rb_queue_render_task(RBQueueArgs *args); * Queue an audio buffer for rendering. The queuing is guaranteed to return * within the given max time budget. The buffer will be dropped if queuing is * not possible within the given time budget. + * Note: Single producer function. Not thread safe, should be called from chain + * function only! * * Convenience function that also handles queuing result by logging if frames * are dropped (DEBUG level). diff --git a/test.ps1 b/test.ps1 index 577f820..74c0187 100644 --- a/test.ps1 +++ b/test.ps1 @@ -34,7 +34,8 @@ switch ($args) { & gst-launch-1.0 -v ` audiotestsrc ! queue ! audioconvert ! ` projectm ` - ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false + is-live=true ` + ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false break } @@ -43,7 +44,8 @@ switch ($args) { & gst-launch-1.0 -v ` audiotestsrc ! queue ! audioconvert ! ` projectm preset="test/presets/215-wave.milk" ` - ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false + is-live=true ` + ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false break } @@ -63,7 +65,8 @@ switch ($args) { mesh-size="512,512" ` easter-egg=0.75 ` preset-locked=false ` - ! "video/x-raw,width=512,height=512,framerate=30/1" ! videoconvert ! xvimagesink sync=false + is-live=true ` + ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=30/1" ! glimagesink sync=false break } @@ -71,7 +74,7 @@ switch ($args) { $env:GST_DEBUG = "3" & gst-launch-1.0 -v ` filesrc location="test/audio/upbeat-future-bass.mp3" ! decodebin ! audioconvert ! ` - projectm ! videoscale ! videoconvert ! video/x-raw,width=1280,height=720 ! ` + projectm ! video/x-raw(memory:GLMemory),width=1280,height=720 ! gldownload ! videoscale ! videoconvert` x264enc ! mp4mux ! filesink location="test/output/test_video.mp4" break } @@ -82,7 +85,7 @@ switch ($args) { filesrc location="test/audio/upbeat-future-bass.mp3" ! decodebin name=dec ! ` audioconvert ! avenc_aac ! avmux_mp4 ! filesink location="test/output/video2.mp4" ` dec. ! ` - projectm ! videoconvert ! x264enc ! avenc_mp4 ! avmux_mp4.video_0 + projectm ! gldownload ! videoconvert ! x264enc ! avenc_mp4 ! avmux_mp4.video_0 break } diff --git a/test.sh b/test.sh index c3a3750..3585ff6 100755 --- a/test.sh +++ b/test.sh @@ -4,10 +4,8 @@ set -e # Set variables based on OS if [[ "$OSTYPE" == "linux-gnu"* ]]; then LIB_EXT="so" - VIDEO_SINK="xvimagesink" elif [[ "$OSTYPE" == "darwin"* ]]; then LIB_EXT="dylib" - VIDEO_SINK="osxvideosink" else echo "Unsupported OS!" exit 1 @@ -52,14 +50,16 @@ case "$1" in GST_DEBUG=projectm:5 gst-launch-1.0 -v \ audiotestsrc ! queue ! audioconvert ! \ projectm \ - ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! $VIDEO_SINK sync=false + is-live=true \ + ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false ;; "--preset") GST_DEBUG=4 gst-launch-1.0 -v \ audiotestsrc ! queue ! audioconvert ! \ projectm preset="test/presets/250-wavecode.milk.milk" \ - ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! $VIDEO_SINK sync=false + is-live=true \ + ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false ;; "--properties") @@ -77,13 +77,14 @@ case "$1" in mesh-size="512,512" \ easter-egg=0.75 \ preset-locked=false \ - ! "video/x-raw,width=512,height=512,framerate=30/1" ! videoconvert ! $VIDEO_SINK sync=false + is-live=true \ + ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=30/1" ! glimagesink sync=false ;; "--output-video") GST_DEBUG=3 gst-launch-1.0 -v \ filesrc location="test/audio/upbeat-future-bass.mp3" ! decodebin ! audioconvert ! \ - projectm preset="test/presets/250-wavecode.milk.milk" ! videoscale ! videoconvert ! video/x-raw,width=1280,height=720 ! \ + projectm preset="test/presets/250-wavecode.milk.milk" ! "video/x-raw(memory:GLMemory),width=1280,height=720" ! gldownload ! videoscale ! videoconvert \ x264enc ! mp4mux ! filesink location="test/output/test_video.mp4" ;; @@ -92,7 +93,7 @@ case "$1" in filesrc location="test/audio/upbeat-future-bass.mp3" ! decodebin name=dec ! \ audioconvert ! avenc_aac ! avmux_mp4 ! filesink location="test/output/video2.mp4" \ dec. ! \ - projectm preset="test/presets/250-wavecode.milk.milk" ! videoconvert ! x264enc ! avenc_mp4 ! avmux_mp4.video_0 + projectm preset="test/presets/250-wavecode.milk.milk" ! gldownload ! videoconvert ! x264enc ! avenc_mp4 ! avmux_mp4.video_0 ;; *) From c26763ca8bdb8f91560c42c9f0d2467321b85162 Mon Sep 17 00:00:00 2001 From: Michael Baetgen Date: Wed, 29 Oct 2025 00:05:13 -0500 Subject: [PATCH 32/32] fix pool shutdown before gl stop --- src/gstglbaseaudiovisualizer.c | 10 ++++++---- src/gstpmaudiovisualizer.c | 14 ++++---------- src/gstpmaudiovisualizer.h | 3 +++ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/gstglbaseaudiovisualizer.c b/src/gstglbaseaudiovisualizer.c index 37b7c04..fe6bfe8 100644 --- a/src/gstglbaseaudiovisualizer.c +++ b/src/gstglbaseaudiovisualizer.c @@ -122,7 +122,7 @@ G_DEFINE_ABSTRACT_TYPE_WITH_CODE( "glbaseaudiovisualizer", 0, "glbaseaudiovisualizer element");); -static void gst_gl_base_audio_visualizer_finalize(GObject *object); +static void gst_gl_base_audio_visualizer_dispose(GObject *object); static void gst_gl_base_audio_visualizer_set_property(GObject *object, guint prop_id, const GValue *value, @@ -222,7 +222,7 @@ gst_gl_base_audio_visualizer_class_init(GstGLBaseAudioVisualizerClass *klass) { GstPMAudioVisualizerClass *pmav_class = GST_PM_AUDIO_VISUALIZER_CLASS(klass); GstElementClass *element_class = GST_ELEMENT_CLASS(klass); - gobject_class->finalize = gst_gl_base_audio_visualizer_finalize; + gobject_class->dispose = gst_gl_base_audio_visualizer_dispose; gobject_class->set_property = gst_gl_base_audio_visualizer_set_property; gobject_class->get_property = gst_gl_base_audio_visualizer_get_property; @@ -313,13 +313,13 @@ static void gst_gl_base_audio_visualizer_init(GstGLBaseAudioVisualizer *glav) { gst_gl_base_audio_visualizer_start(glav); } -static void gst_gl_base_audio_visualizer_finalize(GObject *object) { +static void gst_gl_base_audio_visualizer_dispose(GObject *object) { GstGLBaseAudioVisualizer *glav = GST_GL_BASE_AUDIO_VISUALIZER(object); gst_gl_base_audio_visualizer_stop(glav); g_rec_mutex_clear(&glav->priv->context_lock); - G_OBJECT_CLASS(parent_class)->finalize(object); + G_OBJECT_CLASS(parent_class)->dispose(object); } static void gst_gl_base_audio_visualizer_set_property(GObject *object, @@ -543,6 +543,8 @@ static void gst_gl_base_audio_visualizer_gl_stop(GstGLContext *context, if (glav->priv->fbo) { gst_object_unref(glav->priv->fbo); } + + gst_pm_audio_visualizer_dispose_buffer_pool(GST_PM_AUDIO_VISUALIZER(data)); } static gboolean gst_gl_base_audio_visualizer_default_fill_gl_memory( diff --git a/src/gstpmaudiovisualizer.c b/src/gstpmaudiovisualizer.c index 703a35b..56a2ee9 100644 --- a/src/gstpmaudiovisualizer.c +++ b/src/gstpmaudiovisualizer.c @@ -1187,16 +1187,6 @@ gst_pm_audio_visualizer_parent_change_state(GstElement *element, if (ret == GST_STATE_CHANGE_FAILURE) return ret; - switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: - gst_pm_audio_visualizer_set_allocation(scope, NULL, NULL, NULL, NULL); - break; - case GST_STATE_CHANGE_READY_TO_NULL: - break; - default: - break; - } - GstPMAudioVisualizerClass *klass = GST_PM_AUDIO_VISUALIZER_GET_CLASS(scope); return klass->change_state(element, transition); } @@ -1255,3 +1245,7 @@ void gst_pm_audio_visualizer_adjust_fps(GstPMAudioVisualizer *scope, g_idle_add(log_fps_change, message); } } + +void gst_pm_audio_visualizer_dispose_buffer_pool(GstPMAudioVisualizer* scope) { + gst_pm_audio_visualizer_set_allocation(scope, NULL, NULL, NULL, NULL); +} diff --git a/src/gstpmaudiovisualizer.h b/src/gstpmaudiovisualizer.h index 7d31794..2602dd3 100644 --- a/src/gstpmaudiovisualizer.h +++ b/src/gstpmaudiovisualizer.h @@ -169,6 +169,9 @@ gst_pm_audio_visualizer_util_prepare_output_buffer(GstPMAudioVisualizer *scope, void gst_pm_audio_visualizer_adjust_fps(GstPMAudioVisualizer *scope, guint64 frame_duration); +void +gst_pm_audio_visualizer_dispose_buffer_pool(GstPMAudioVisualizer *scope); + G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstPMAudioVisualizer, gst_object_unref) G_END_DECLS