I’m working on a project where I want to dynamically inject an image (e.g., a red image) into a GStreamer video stream using appsrc. My goal is to send this image at 30 fps while maintaining a final pipeline output at 60 fps.

I’ve written some test code, but I’m encountering issues when pushing a buffer to appsrc. I’m not sure if the appsrc caps are configured correctly or if I’m handling buffers and timestamps properly.

Here’s what I have so far:

import cv2
import numpy as np
import gi
import time
import threading
from fractions import Fraction

gi.require_version('Gst', '1.0')
from gi.repository import Gst, GLib

Gst.init(None)

pipeline_description = """

    filesrc location="overlay.png" ! pngdec ! videoconvert ! video/x-raw,format=NV12 ! imagefreeze ! queue max-size-buffers=1 max-size-time=0 max-size-bytes=0 ! compositor.sink_0 \

    appsrc name=display_screen ! queue max-size-buffers=4 ! compositor.sink_3 \

    v4l2src device=/dev/video0 name=game_capture io-mode=2 ! video/x-raw,width=720,height=480,format=NV12 ! videoscale method=1 ! video/x-raw,width=864,height=576 ! queue max-size-buffers=1 max-size-time=0 max-size-bytes=0 ! compositor.sink_1 \

    v4l2src device=/dev/video2 name=webcam io-mode=2 ! video/x-raw,width=320,height=240 ! videoscale method=1 ! video/x-raw,width=324,height=244 ! videocrop top=0 bottom=74 left=0 right=0 ! video/x-raw,width=324,height=170 ! queue max-size-buffers=1 max-size-time=0 max-size-bytes=0 ! compositor.sink_2 \

    compositor name=compositor sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=25 sink_1::ypos=59 sink_2::xpos=930 sink_2::ypos=104 sink_3::xpos=126 sink_3::ypos=641 ! video/x-raw,format=NV12,width=1280,height=720,framerate=60/1 ! \
    x264enc bitrate=4000  tune=zerolatency speed-preset=superfast ! h264parse ! queue max-size-buffers=1 max-size-time=0 max-size-bytes=0 ! mux.video \

    alsasrc device=hw:2 ! audioconvert ! audioresample ! audio/x-raw,channels=2,rate=44100 ! queue max-size-buffers=1 max-size-time=0 max-size-bytes=0 ! faac bitrate=128000 ! mux.audio \

    flvmux name=mux latency=0 ! filesink location="stream.flv"
"""

pipeline = Gst.parse_launch(pipeline_description)

# Start the pipeline
pipeline.set_state(Gst.State.PLAYING)

# Retrieve the camera element
appsrc = pipeline.get_by_name('display_screen')
appsrc.set_property('caps', Gst.Caps.from_string("video/x-raw,format=RGB,width=667,height=74,framerate=30/1"))
appsrc.set_property("format", Gst.Format.TIME)  
appsrc.set_property("block", True)

red_image = np.zeros((74, 667, 3), dtype=np.uint8)
red_image[:, :, 0] = 0 
red_image[:, :, 1] = 0 
red_image[:, :, 2] = 255

bus = pipeline.get_bus()

def push_frame_to_appsrc():
    pts = 0
    FPS = Fraction(30)
    duration = 10**9 / (FPS.numerator / FPS.denominator) 
    while True:

        gst_buffer = Gst.Buffer.new_wrapped(red_image.tobytes())

        pts += duration 
        gst_buffer.pts = pts
        gst_buffer.duration = duration

        appsrc.emit("push-buffer", gst_buffer)

def message():
    while True:
        message = bus.timed_pop_filtered(1000000000, Gst.MessageType.ERROR | Gst.MessageType.EOS)
        if message:
            print(message.type)
            if message.type == Gst.MessageType.ERROR:
                err, dbg = message.parse_error()
                print("Error:", err, dbg)
            elif message.type == Gst.MessageType.EOS:
                print("End of stream")
                break

try:

    bt = threading.Thread(target=message)
    bt.daemon = True
    bt.start()

    appt = threading.Thread(target=push_frame_to_appsrc)
    appt.daemon = True
    appt.start()

    loop = GLib.MainLoop()
    loop.run()
    

except KeyboardInterrupt:
    pass
finally:
    # Cleanup
    pipeline.set_state(Gst.State.NULL)

Problem

• When pushing buffers using appsrc.emit("push-buffer", gst_buffer), I encounter errors.
• The error messages from the bus are not always clear.
• I’m unsure if the appsrc caps are properly configured.
• I’m uncertain about handling the pts (timestamps) and buffer durations, considering the pipeline outputs at 60 fps while my injected images are at 30 fps.

What I’m Looking For

• A review of my appsrc configuration, especially the caps.
• Advice or a working example to correctly handle timestamps and buffer durations in this context.
• If possible, suggestions on how to synchronize a 30 fps injected stream with a 60 fps final output in GStreamer.

Thank you in advance for your help and insights!