GStreamer-devel

Frame Stepping

classic Classic list List threaded Threaded
♦
♦
 2 messages Options
Brian Panneton
Reply | Threaded
Open this post in threaded view
♦
♦
|

Frame Stepping

Brian Panneton
All,

I've been trying to step through frame by frame with one pipeline and pass through to another. Basically I want to look at every h264 frame and then pipe it off into a filesink. I have made an example of what I thought would work below.

It should take an MKV file, step thought the h264 frames and pipe them to a MP4 file without missing any frames (It is important that I get the actual frames here). So the first pipeline is in PAUSE mode and the second pipeline is in PLAYING (I don't think writing to a file can be done in paused mode?). I understand I could do this by removing the appsink->appsrc but that is not what I am looking for and wont work in my application.

My big problem at the moment is that I can't seem to determine when the end-of-stream occurs since I am in paused mode. My best attempt was to use pos > dur, but I have found this isn't always true. In the current case I either end early or never end (when I remove the pos == last section). If there is a better way to do this I would appreciate any ideas. Sample code is below! Sample MKV file: http://jell.yfish.us/media/jellyfish-25-mbps-hd-h264.mkv

<code>
from gi.repository import Gst, GObject
import sys
import os
import glob

def test_file(inc):
    filename = "jellyfish-25-mbps-hd-h264.mkv"
    elements = {}

    def demuxer_callback(demuxer, pad):
        if pad.get_property("template").name_template == "video_%u":
            pad.link(elements['h264parse'].get_static_pad("sink"))

    #GObject.threads_init()
    Gst.init()
    Gst.debug_set_active(True)
    Gst.debug_set_default_threshold(1)
    pipeline_front = Gst.Pipeline()
    pipeline_end = Gst.Pipeline()

    elements['filesrc'] = Gst.ElementFactory.make('filesrc', str(inc)+'_filesrc')
    elements['filesrc'].set_property('location', filename)
    elements['matroskademux'] = Gst.ElementFactory.make('matroskademux', str(inc)+'_matroskademux')
    elements['matroskademux'].connect('pad-added', demuxer_callback)
    elements['h264parse'] = Gst.ElementFactory.make('h264parse', str(inc)+'_h264parse')
    elements['raw1queue'] = Gst.ElementFactory.make('queue', str(inc)+'_raw1queue')
    elements['innersink'] = Gst.ElementFactory.make('appsink', str(inc)+'_appsink')

    elements['mp4appsrc'] = Gst.ElementFactory.make('appsrc')
    elements['mp4h264parse'] = Gst.ElementFactory.make('h264parse')
    elements['mp4queue'] = Gst.ElementFactory.make('queue')
    elements['mp4mux'] = Gst.ElementFactory.make('mp4mux')
    elements['mp4mqueue'] = Gst.ElementFactory.make('queue')
    elements['mp4filesink'] = Gst.ElementFactory.make('filesink')
    elements['mp4filesink'].set_property('location', "{}.mp4".format(inc))

    pipeline_front.add(elements['filesrc'])
    pipeline_front.add(elements['matroskademux'])
    pipeline_front.add(elements['h264parse'])
    pipeline_front.add(elements['raw1queue'])
    pipeline_front.add(elements['innersink'])

    pipeline_end.add(elements['mp4appsrc'])
    pipeline_end.add(elements['mp4h264parse'])
    pipeline_end.add(elements['mp4queue'])
    pipeline_end.add(elements['mp4mux'])
    pipeline_end.add(elements['mp4mqueue'])
    pipeline_end.add(elements['mp4filesink'])

    elements['filesrc'].link(elements['matroskademux'])
    elements['matroskademux'].link(elements['h264parse'])
    elements['h264parse'].link(elements['raw1queue'])
    elements['raw1queue'].link(elements['innersink'])

    elements['mp4appsrc'].link(elements['mp4queue'])
    elements['mp4queue'].link(elements['mp4h264parse'])
    elements['mp4h264parse'].link(elements['mp4mux'])
    elements['mp4mux'].link(elements['mp4mqueue'])
    elements['mp4mqueue'].link(elements['mp4filesink'])

    pipeline_front.set_state(Gst.State.READY)
    pipeline_front.set_state(Gst.State.PAUSED)

    pipeline_end.set_state(Gst.State.READY)
    pipeline_end.set_state(Gst.State.PAUSED)
    pipeline_end.set_state(Gst.State.PLAYING)
    print 'Front Setup Status:', pipeline_front.get_state(Gst.CLOCK_TIME_NONE)[0]
    print 'Start', inc
    completed = False
    last = None
    while True:
        if not completed:
            pipeline_front.get_state(Gst.CLOCK_TIME_NONE)[0]
            elements['innersink'].send_event(Gst.Event.new_step(Gst.Format.BUFFERS, 1, 1, True, False))
            inner_sample = elements['innersink'].emit('pull-preroll')
            inner_buffer = inner_sample.get_buffer()
            elements['mp4appsrc'].set_property('caps', inner_sample.get_caps())
            elements['mp4appsrc'].emit('push-buffer', inner_buffer)

        # Check position for completion
        # Can't figure out how to tell there are no more frames
        pos = elements['innersink'].query_position(Gst.Format.TIME)[1]
        dur = elements['innersink'].query_duration(Gst.Format.TIME)[1]
        print 'Loc:', pos, dur

        if pos > dur:
            completed = True
        if pos == last:
            completed = True

        bus = pipeline_front.get_bus()
        while True:
            message = bus.pop_filtered(Gst.MessageType.ANY)
            if not message:
                break
            elif message.type in [Gst.MessageType.EOS]:
                completed = True
                break

        # Close stream if completed
        if completed:
            elements['mp4appsrc'].emit('end-of-stream')
            break
        last = pos

    pipeline_front.set_state(Gst.State.NULL)
    pipeline_end.set_state(Gst.State.NULL)
    print 'End', inc, pos, dur

test_file(0)
</code>

Thanks,
Brian



_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel
Brian Panneton
Reply | Threaded
Open this post in threaded view
♦
♦
|

Re: Frame Stepping

Brian Panneton
I realized that I'm ending early with pos == last because multiple frames seem to have the same date. Is this supposed to happen? I'm really hoping there is an easy way to determine the end of the stream while in PAUSED mode.

Thanks,
Brian

On Thu, Aug 11, 2016 at 5:10 PM, Brian Panneton <[hidden email]> wrote:
All,

I've been trying to step through frame by frame with one pipeline and pass through to another. Basically I want to look at every h264 frame and then pipe it off into a filesink. I have made an example of what I thought would work below.

It should take an MKV file, step thought the h264 frames and pipe them to a MP4 file without missing any frames (It is important that I get the actual frames here). So the first pipeline is in PAUSE mode and the second pipeline is in PLAYING (I don't think writing to a file can be done in paused mode?). I understand I could do this by removing the appsink->appsrc but that is not what I am looking for and wont work in my application.

My big problem at the moment is that I can't seem to determine when the end-of-stream occurs since I am in paused mode. My best attempt was to use pos > dur, but I have found this isn't always true. In the current case I either end early or never end (when I remove the pos == last section). If there is a better way to do this I would appreciate any ideas. Sample code is below! Sample MKV file: http://jell.yfish.us/media/jellyfish-25-mbps-hd-h264.mkv

<code>
from gi.repository import Gst, GObject
import sys
import os
import glob

def test_file(inc):
    filename = "jellyfish-25-mbps-hd-h264.mkv"
    elements = {}

    def demuxer_callback(demuxer, pad):
        if pad.get_property("template").name_template == "video_%u":
            pad.link(elements['h264parse'].get_static_pad("sink"))

    #GObject.threads_init()
    Gst.init()
    Gst.debug_set_active(True)
    Gst.debug_set_default_threshold(1)
    pipeline_front = Gst.Pipeline()
    pipeline_end = Gst.Pipeline()

    elements['filesrc'] = Gst.ElementFactory.make('filesrc', str(inc)+'_filesrc')
    elements['filesrc'].set_property('location', filename)
    elements['matroskademux'] = Gst.ElementFactory.make('matroskademux', str(inc)+'_matroskademux')
    elements['matroskademux'].connect('pad-added', demuxer_callback)
    elements['h264parse'] = Gst.ElementFactory.make('h264parse', str(inc)+'_h264parse')
    elements['raw1queue'] = Gst.ElementFactory.make('queue', str(inc)+'_raw1queue')
    elements['innersink'] = Gst.ElementFactory.make('appsink', str(inc)+'_appsink')

    elements['mp4appsrc'] = Gst.ElementFactory.make('appsrc')
    elements['mp4h264parse'] = Gst.ElementFactory.make('h264parse')
    elements['mp4queue'] = Gst.ElementFactory.make('queue')
    elements['mp4mux'] = Gst.ElementFactory.make('mp4mux')
    elements['mp4mqueue'] = Gst.ElementFactory.make('queue')
    elements['mp4filesink'] = Gst.ElementFactory.make('filesink')
    elements['mp4filesink'].set_property('location', "{}.mp4".format(inc))

    pipeline_front.add(elements['filesrc'])
    pipeline_front.add(elements['matroskademux'])
    pipeline_front.add(elements['h264parse'])
    pipeline_front.add(elements['raw1queue'])
    pipeline_front.add(elements['innersink'])

    pipeline_end.add(elements['mp4appsrc'])
    pipeline_end.add(elements['mp4h264parse'])
    pipeline_end.add(elements['mp4queue'])
    pipeline_end.add(elements['mp4mux'])
    pipeline_end.add(elements['mp4mqueue'])
    pipeline_end.add(elements['mp4filesink'])

    elements['filesrc'].link(elements['matroskademux'])
    elements['matroskademux'].link(elements['h264parse'])
    elements['h264parse'].link(elements['raw1queue'])
    elements['raw1queue'].link(elements['innersink'])

    elements['mp4appsrc'].link(elements['mp4queue'])
    elements['mp4queue'].link(elements['mp4h264parse'])
    elements['mp4h264parse'].link(elements['mp4mux'])
    elements['mp4mux'].link(elements['mp4mqueue'])
    elements['mp4mqueue'].link(elements['mp4filesink'])

    pipeline_front.set_state(Gst.State.READY)
    pipeline_front.set_state(Gst.State.PAUSED)

    pipeline_end.set_state(Gst.State.READY)
    pipeline_end.set_state(Gst.State.PAUSED)
    pipeline_end.set_state(Gst.State.PLAYING)
    print 'Front Setup Status:', pipeline_front.get_state(Gst.CLOCK_TIME_NONE)[0]
    print 'Start', inc
    completed = False
    last = None
    while True:
        if not completed:
            pipeline_front.get_state(Gst.CLOCK_TIME_NONE)[0]
            elements['innersink'].send_event(Gst.Event.new_step(Gst.Format.BUFFERS, 1, 1, True, False))
            inner_sample = elements['innersink'].emit('pull-preroll')
            inner_buffer = inner_sample.get_buffer()
            elements['mp4appsrc'].set_property('caps', inner_sample.get_caps())
            elements['mp4appsrc'].emit('push-buffer', inner_buffer)

        # Check position for completion
        # Can't figure out how to tell there are no more frames
        pos = elements['innersink'].query_position(Gst.Format.TIME)[1]
        dur = elements['innersink'].query_duration(Gst.Format.TIME)[1]
        print 'Loc:', pos, dur

        if pos > dur:
            completed = True
        if pos == last:
            completed = True

        bus = pipeline_front.get_bus()
        while True:
            message = bus.pop_filtered(Gst.MessageType.ANY)
            if not message:
                break
            elif message.type in [Gst.MessageType.EOS]:
                completed = True
                break

        # Close stream if completed
        if completed:
            elements['mp4appsrc'].emit('end-of-stream')
            break
        last = pos

    pipeline_front.set_state(Gst.State.NULL)
    pipeline_end.set_state(Gst.State.NULL)
    print 'End', inc, pos, dur

test_file(0)
</code>

Thanks,
Brian




_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel
Free forum by Nabble Edit this page