I'm using FFmpeg.AutoGen in a .NET 8.0 application for decoding H264 and JPEG.

Every usage uses its own instance (there are no shared objects) and there isn't simultanous use of each object.

However, they can be used from different threads.

But I'm getting the encoded results with mixed data from all sources. It happens mostly for the same origin codeec, but not esclusivelly.

I have a base class that looks like this:

public abstract unsafe class Decoder : IDisposable
{
    protected readonly object sync = new();
    protected AVCodecContext* codecContext;
    protected AVFrame* frame;
    protected AVPacket* packet;
    protected SwsContext* swsContext;
    private bool disposed = false;

    protected Decoder(AVCodecID codecId, ChannelWriter<(ArraySegment<byte> decodedData, int width, int height)> writer)
    {
        Writer = writer;
        var codec = ffmpeg.avcodec_find_decoder(codecId);
        codecContext = ffmpeg.avcodec_alloc_context3(codec);
        ffmpeg.avcodec_open2(codecContext, codec, null);

        frame = ffmpeg.av_frame_alloc();
        packet = ffmpeg.av_packet_alloc();
    }
...

This is the H264 decoding code:

public override void Decode(ReadOnlySpan<byte> data)
{
    CheckDisposed();

    lock (sync)
    {
        fixed (byte* pData = data)
        {
            ffmpeg.av_packet_unref(packet);

            packet->data = pData;
            packet->size = data.Length;

            var result = ffmpeg.avcodec_send_packet(codecContext, packet);
            if (result < 0 && result != ffmpeg.AVERROR(ffmpeg.EAGAIN))
            {
                EvalResult(result);
            }

            while (true)
            {
                ffmpeg.av_frame_unref(frame);

                result = ffmpeg.avcodec_receive_frame(codecContext, frame);
                if (result == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // Need more input data, continue sending packets
                    return;
                }
                else if (result < 0)
                {
                    EvalResult(result);
                }

                var width = frame->width;
                var height = frame->height;

                if (swsContext == null)
                {
                    swsContext = ffmpeg.sws_getContext(
                        width, height, (AVPixelFormat)frame->format,
                        width, height, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                var bgraFrame = ffmpeg.av_frame_alloc();
                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = width;
                bgraFrame->height = height;
                ffmpeg.av_frame_get_buffer(bgraFrame, 32);

                ffmpeg.sws_scale(
                    swsContext,
                    frame->data, frame->linesize, 0, height,
                    bgraFrame->data, bgraFrame->linesize);

                var bgraSize = width * height * 4;
                var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
                var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);

                fixed (byte* pBGRAData = bgraSegment.Array)
                {
                    var data4 = new byte_ptr4();
                    data4.UpdateFrom(bgraFrame->data.ToArray());

                    var linesize4 = new int4();
                    linesize4.UpdateFrom(bgraFrame->linesize.ToArray());

                    ffmpeg.av_image_copy_to_buffer(
                        pBGRAData, bgraSize,
                        data4, linesize4,
                        (AVPixelFormat)bgraFrame->format, width, height, 1);
                }

                ffmpeg.av_frame_free(&bgraFrame);

                Writer.TryWrite((bgraSegment, width, height));
            }
        }
    }
}

And this is the JPEG decoding code:

public override void Decode(ReadOnlySpan<byte> data)
{
    CheckDisposed();

    lock (sync)
    {
        fixed (byte* pData = data)
        {
            ffmpeg.av_packet_unref(packet);

            packet->data = pData;
            packet->size = data.Length;

            var result = ffmpeg.avcodec_send_packet(codecContext, packet);
            if (result < 0 && result != ffmpeg.AVERROR(ffmpeg.EAGAIN))
            {
                EvalResult(result);
            }

            while (true)
            {
                ffmpeg.av_frame_unref(frame);

                result = ffmpeg.avcodec_receive_frame(codecContext, frame);

                if (result == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // Need more input data, continue sending packets
                    return;
                }
                else if (result < 0)
                {
                    EvalResult(result);
                }

                var width = frame->width;
                var height = frame->height;

                if (swsContext == null)
                {
                    swsContext = ffmpeg.sws_getContext(
                        width, height, (AVPixelFormat)frame->format,
                        width, height, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                var bgraFrame = ffmpeg.av_frame_alloc();
                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = frame->width;
                bgraFrame->height = frame->height;
                ffmpeg.av_frame_get_buffer(bgraFrame, 32);

                ffmpeg.sws_scale(
                    swsContext,
                    frame->data, frame->linesize, 0, frame->height,
                    bgraFrame->data, bgraFrame->linesize);

                int bgraDataSize = bgraFrame->linesize[0] * bgraFrame->height;
                byte[] bgraData = ArrayPool<byte>.Shared.Rent(bgraDataSize);
                var bgraSegment = new ArraySegment<byte>(bgraData, 0, bgraDataSize);
                Marshal.Copy((IntPtr)bgraFrame->data[0], bgraData, 0, bgraDataSize);

                ffmpeg.av_frame_free(&bgraFrame);

                Writer.TryWrite((bgraSegment, width, height));
            }
        }
    }
}

What am I doing wrong here?

I'm using FFmpeg.AutoGen in a .NET 8.0 application for decoding H264 and JPEG.

Every usage uses its own instance (there are no shared objects) and there isn't simultanous use of each object.

However, they can be used from different threads.

But I'm getting the encoded results with mixed data from all sources. It happens mostly for the same origin codeec, but not esclusivelly.

I have a base class that looks like this:

public abstract unsafe class Decoder : IDisposable
{
    protected readonly object sync = new();
    protected AVCodecContext* codecContext;
    protected AVFrame* frame;
    protected AVPacket* packet;
    protected SwsContext* swsContext;
    private bool disposed = false;

    protected Decoder(AVCodecID codecId, ChannelWriter<(ArraySegment<byte> decodedData, int width, int height)> writer)
    {
        Writer = writer;
        var codec = ffmpeg.avcodec_find_decoder(codecId);
        codecContext = ffmpeg.avcodec_alloc_context3(codec);
        ffmpeg.avcodec_open2(codecContext, codec, null);

        frame = ffmpeg.av_frame_alloc();
        packet = ffmpeg.av_packet_alloc();
    }
...

This is the H264 decoding code:

public override void Decode(ReadOnlySpan<byte> data)
{
    CheckDisposed();

    lock (sync)
    {
        fixed (byte* pData = data)
        {
            ffmpeg.av_packet_unref(packet);

            packet->data = pData;
            packet->size = data.Length;

            var result = ffmpeg.avcodec_send_packet(codecContext, packet);
            if (result < 0 && result != ffmpeg.AVERROR(ffmpeg.EAGAIN))
            {
                EvalResult(result);
            }

            while (true)
            {
                ffmpeg.av_frame_unref(frame);

                result = ffmpeg.avcodec_receive_frame(codecContext, frame);
                if (result == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // Need more input data, continue sending packets
                    return;
                }
                else if (result < 0)
                {
                    EvalResult(result);
                }

                var width = frame->width;
                var height = frame->height;

                if (swsContext == null)
                {
                    swsContext = ffmpeg.sws_getContext(
                        width, height, (AVPixelFormat)frame->format,
                        width, height, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                var bgraFrame = ffmpeg.av_frame_alloc();
                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = width;
                bgraFrame->height = height;
                ffmpeg.av_frame_get_buffer(bgraFrame, 32);

                ffmpeg.sws_scale(
                    swsContext,
                    frame->data, frame->linesize, 0, height,
                    bgraFrame->data, bgraFrame->linesize);

                var bgraSize = width * height * 4;
                var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
                var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);

                fixed (byte* pBGRAData = bgraSegment.Array)
                {
                    var data4 = new byte_ptr4();
                    data4.UpdateFrom(bgraFrame->data.ToArray());

                    var linesize4 = new int4();
                    linesize4.UpdateFrom(bgraFrame->linesize.ToArray());

                    ffmpeg.av_image_copy_to_buffer(
                        pBGRAData, bgraSize,
                        data4, linesize4,
                        (AVPixelFormat)bgraFrame->format, width, height, 1);
                }

                ffmpeg.av_frame_free(&bgraFrame);

                Writer.TryWrite((bgraSegment, width, height));
            }
        }
    }
}

And this is the JPEG decoding code:

public override void Decode(ReadOnlySpan<byte> data)
{
    CheckDisposed();

    lock (sync)
    {
        fixed (byte* pData = data)
        {
            ffmpeg.av_packet_unref(packet);

            packet->data = pData;
            packet->size = data.Length;

            var result = ffmpeg.avcodec_send_packet(codecContext, packet);
            if (result < 0 && result != ffmpeg.AVERROR(ffmpeg.EAGAIN))
            {
                EvalResult(result);
            }

            while (true)
            {
                ffmpeg.av_frame_unref(frame);

                result = ffmpeg.avcodec_receive_frame(codecContext, frame);

                if (result == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // Need more input data, continue sending packets
                    return;
                }
                else if (result < 0)
                {
                    EvalResult(result);
                }

                var width = frame->width;
                var height = frame->height;

                if (swsContext == null)
                {
                    swsContext = ffmpeg.sws_getContext(
                        width, height, (AVPixelFormat)frame->format,
                        width, height, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                var bgraFrame = ffmpeg.av_frame_alloc();
                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = frame->width;
                bgraFrame->height = frame->height;
                ffmpeg.av_frame_get_buffer(bgraFrame, 32);

                ffmpeg.sws_scale(
                    swsContext,
                    frame->data, frame->linesize, 0, frame->height,
                    bgraFrame->data, bgraFrame->linesize);

                int bgraDataSize = bgraFrame->linesize[0] * bgraFrame->height;
                byte[] bgraData = ArrayPool<byte>.Shared.Rent(bgraDataSize);
                var bgraSegment = new ArraySegment<byte>(bgraData, 0, bgraDataSize);
                Marshal.Copy((IntPtr)bgraFrame->data[0], bgraData, 0, bgraDataSize);

                ffmpeg.av_frame_free(&bgraFrame);

                Writer.TryWrite((bgraSegment, width, height));
            }
        }
    }
}

What am I doing wrong here?

• c#
• multithreading
• ffmpeg
• thread-safety

• Even moving all ffmpeg processing to its own thread, there's data mixing. Up until ffmpeg.avcodec_receive_frame starts returning only AVERROR(EAGAIN) – Paulo Mado Commented Apr 1 at 13:20
• Eventually, I start getting 0xbebbb1b7. – Paulo Mado Commented Apr 1 at 13:36
• avcodec_decode_video2 works, but it's deprecated. – Paulo Mado Commented Apr 1 at 17:42

1 Answer 1

It turns out I wasn't unreferencing packets and frames:

        private unsafe void HandleEncodedpacket(ArraySegment<byte> encodedData)
        {
            Debug.Assert(_ffmpegContext is not null);
            var codecContext = _ffmpegContext.codecContext;
            var swsContext = _ffmpegContext.swsContext;
            var packet = _ffmpegContext.packet;
            var sourceFrame = _ffmpegContext.sourceFrame;
            var bgraFrame = _ffmpegContext.bgraFrame;
            Debug.Assert(codecContext != null);
            Debug.Assert(packet != null);
            Debug.Assert(sourceFrame != null);
            Debug.Assert(bgraFrame != null);

            int result;

            fixed (byte* encodedDataBytes = encodedData.Array)
            {
                packet->data = encodedDataBytes;
                packet->size = encodedData.Count;

                result =
                ffmpeg.avcodec_send_packet(codecContext, packet)
                    //.ThrowIfError()
                    ;

                ffmpeg.av_packet_unref(packet); // <---

                if (result != 0)
                {
                    return;
                }
            }

            while ((result = ffmpeg.avcodec_receive_frame(codecContext, sourceFrame)) >= 0)
            {
                var sourceWidth = sourceFrame->width;
                var sourceHeight = sourceFrame->height;
                var destinationWidth = sourceFrame->width;
                var destinationHeight = sourceFrame->height;

                // Convert the frame to BGRA
                var pixelFormat = (AVPixelFormat)sourceFrame->format switch
                {
                    AVPixelFormat.AV_PIX_FMT_YUVJ420P => AVPixelFormat.AV_PIX_FMT_YUV420P,
                    AVPixelFormat.AV_PIX_FMT_YUVJ422P => AVPixelFormat.AV_PIX_FMT_YUV422P,
                    AVPixelFormat.AV_PIX_FMT_YUVJ444P => AVPixelFormat.AV_PIX_FMT_YUV444P,
                    _ => (AVPixelFormat)sourceFrame->format,
                };

                if (swsContext == null)
                {
                    _ffmpegContext.swsContext = swsContext = ffmpeg.sws_getContext(
                        sourceWidth, sourceHeight, pixelFormat,
                        destinationWidth, destinationHeight, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = destinationWidth;
                bgraFrame->height = destinationHeight;

                ffmpeg.av_frame_get_buffer(bgraFrame, 32).ThrowIfError();

                ffmpeg.sws_scale(
                    swsContext,
                    sourceFrame->data, sourceFrame->linesize, 0, sourceFrame->height,
                    bgraFrame->data, bgraFrame->linesize);

                ffmpeg.av_frame_unref(sourceFrame); // <---

                var bgraSize = ffmpeg.av_image_get_buffer_size(AVPixelFormat.AV_PIX_FMT_BGRA, destinationWidth, destinationHeight, 1);
                var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
                var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);

                fixed (byte* pBgraData = bgraArray)
                {
                    var data4 = new byte_ptrArray4();
                    data4.UpdateFrom(bgraFrame->data.ToArray());

                    var linesize4 = new int_array4();
                    linesize4.UpdateFrom(bgraFrame->linesize.ToArray());

                    ffmpeg.av_image_copy_to_buffer(
                        pBgraData, bgraSize,
                        data4, linesize4,
                        (AVPixelFormat)bgraFrame->format, destinationWidth, destinationHeight, 1);
                }

                ffmpeg.av_frame_unref(bgraFrame); // <---

                _decodedWriter.TryWrite((bgraSegment, destinationWidth, destinationHeight));
            }

            switch (result)
            {
                case 0:
                case var eagain when (eagain == FfmpegContext.eagain):
                case var eof when (eof == ffmpeg.AVERROR_EOF):
                    break;
                default:
                    result.ThrowIfError();
                    break;
            }
        }