appsrc with GST_FORMAT_BYTES cannot query duration

[gyunaev]

Good morning,

I've set up appsrc with format GST_FORMAT_BYTES, use push signal and set the buffer position via buffer->offset and buffer->offset_end. It works fine, but the maximum duration is reported.

However if I also set the "size" property of appsrc, querying the duration fails - gst_element_query_duration() returns 0.

The following example is basically playback-tutorial-3.c with following modifications:

- "format" is set to GST_FORMAT_BYTES, "stream-type" to 1 (seekable) and "size" to 40000000
- added seek callback
- added timeout to print position/duration (which fails on querying duration)
- stream position is set via:
  buffer->offset = data->streampos;
  buffer->offset_end = data->streampos + CHUNK_SIZE - 1;


The sample code (also pasted as http://pastebin.com/ubrWaKQ0 in case mailer breaks it):

#include <gst/gst.h>
#include <gst/audio/audio.h>
#include <string.h>
 
#define CHUNK_SIZE 1024   /* Amount of bytes we are sending in each buffer */
#define SAMPLE_RATE 44100 /* Samples per second we are sending */
 
/* Structure to contain all our information, so we can pass it to callbacks */
typedef struct _CustomData {
  GstElement *pipeline;
  GstElement *app_source;
 
  guint64 num_samples;   /* Number of samples generated so far (for timestamp generation) */
  guint streampos;
  gfloat a, b, c, d;     /* For waveform generation */
 
  guint sourceid;        /* To control the GSource */
 
  GMainLoop *main_loop;  /* GLib's Main Loop */
} CustomData;
 
/* This method is called by the idle GSource in the mainloop, to feed CHUNK_SIZE bytes into appsrc.
 * The ide handler is added to the mainloop when appsrc requests us to start sending data (need-data signal)
 * and is removed when appsrc has enough data (enough-data signal).
 */
static gboolean push_data (CustomData *data) {
  GstBuffer *buffer;
  GstFlowReturn ret;
  int i;
  GstMapInfo map;
  gint16 *raw;
  gint num_samples = CHUNK_SIZE / 2; /* Because each sample is 16 bits */
  gfloat freq;
 
  /* Create a new empty buffer */
  buffer = gst_buffer_new_and_alloc (CHUNK_SIZE);
   
  /* Generate some psychodelic waveforms */
  gst_buffer_map (buffer, &map, GST_MAP_WRITE);
  raw = (gint16 *)map.data;
  data->c += data->d;
  data->d -= data->c / 1000;
  freq = 1100 + 1000 * data->d;
  for (i = 0; i < num_samples; i++) {
    data->a += data->b;
    data->b -= data->a / freq;
    raw[i] = (gint16)(500 * data->a);
  }
  gst_buffer_unmap (buffer, &map);
  data->num_samples += num_samples;
 
  /* Set its timestamp and duration */
  buffer->offset = data->streampos;
  buffer->offset_end = data->streampos + CHUNK_SIZE - 1;
  data->streampos += CHUNK_SIZE;
 
  /* Push the buffer into the appsrc */
  g_signal_emit_by_name (data->app_source, "push-buffer", buffer, &ret);
 
  /* Free the buffer now that we are done with it */
  gst_buffer_unref (buffer);
 
  if (ret != GST_FLOW_OK) {
    /* We got some error, stop sending data */
    return FALSE;
  }
 
  return TRUE;
}
 
/* This signal callback triggers when appsrc needs data. Here, we add an idle handler
 * to the mainloop to start pushing data into the appsrc */
static void start_feed (GstElement *source, guint size, CustomData *data) {
  if (data->sourceid == 0) {
    g_print ("Start feeding\n");
    data->sourceid = g_idle_add ((GSourceFunc) push_data, data);
  }
}
 
/* This callback triggers when appsrc has enough data and we can stop sending.
 * We remove the idle handler from the mainloop */
static void stop_feed (GstElement *source, CustomData *data) {
  if (data->sourceid != 0) {
    g_print ("Stop feeding\n");
    g_source_remove (data->sourceid);
    data->sourceid = 0;
  }
}
 
static gboolean seek_feed(GstElement *sourc, guint64 offset, gpointer user_data)
{
    g_print ("seek called to %d\n", offset);
    return 1;
}

/* This function is called when an error message is posted on the bus */
static void error_cb (GstBus *bus, GstMessage *msg, CustomData *data) {
  GError *err;
  gchar *debug_info;
 
  /* Print error details on the screen */
  gst_message_parse_error (msg, &err, &debug_info);
  g_printerr ("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
  g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
  g_clear_error (&err);
  g_free (debug_info);
 
  g_main_loop_quit (data->main_loop);
}
 
/* This function is called when playbin has created the appsrc element, so we have
 * a chance to configure it. */
static void source_setup (GstElement *pipeline, GstElement *source, CustomData *data) {
  GstAudioInfo info;
  GstCaps *audio_caps;
 
  g_print ("Source has been created. Configuring.\n");
  data->app_source = source;
 
  /* Configure appsrc */
  gst_audio_info_set_format (&info, GST_AUDIO_FORMAT_S16, SAMPLE_RATE, 1, NULL);
  audio_caps = gst_audio_info_to_caps (&info);

  // This does not work
  g_object_set (source, "caps", audio_caps, "format", GST_FORMAT_BYTES, "stream-type", 1, "size", (gint64) 40000000, NULL);
 
  // This works but the duration is not set
  //g_object_set (source, "caps", audio_caps, "format", GST_FORMAT_BYTES, "stream-type", 1, NULL);
  g_signal_connect (source, "need-data", G_CALLBACK (start_feed), data);
  g_signal_connect (source, "enough-data", G_CALLBACK (stop_feed), data);
  g_signal_connect (source, "seek-data", G_CALLBACK (seek_feed), &data);
  gst_caps_unref (audio_caps);
}

static gboolean
cb_print_position (GstElement *pipeline)
{
    gint64 pos, len = -1;
   
    if ( !gst_element_query_position (pipeline, GST_FORMAT_TIME, &pos) )
        g_print ("Failed to query position.\n");
    else if ( !gst_element_query_duration (pipeline, GST_FORMAT_TIME, &len) )
        g_print ("Failed to query duration.\n");
    else
        g_print ("Time: %" GST_TIME_FORMAT " / %" GST_TIME_FORMAT "\r", GST_TIME_ARGS (pos), GST_TIME_ARGS (len));

    /* call me again */
    return TRUE;
}


int main(int argc, char *argv[]) {
  CustomData data;
  GstBus *bus;
 
  /* Initialize cumstom data structure */
  memset (&data, 0, sizeof (data));
  data.b = 1; /* For waveform generation */
  data.d = 1;
 
  /* Initialize GStreamer */
  gst_init (&argc, &argv);
 
  /* Create the playbin element */
  data.pipeline = gst_parse_launch ("playbin uri=appsrc://", NULL);
  g_signal_connect (data.pipeline, "source-setup", G_CALLBACK (source_setup), &data);
 
  /* Instruct the bus to emit signals for each received message, and connect to the interesting signals */
  bus = gst_element_get_bus (data.pipeline);
  gst_bus_add_signal_watch (bus);
  g_signal_connect (G_OBJECT (bus), "message::error", (GCallback)error_cb, &data);
  gst_object_unref (bus);
 
  /* Start playing the pipeline */
  gst_element_set_state (data.pipeline, GST_STATE_PLAYING);
 
  g_timeout_add (200, (GSourceFunc) cb_print_position, data.pipeline);
 
  /* Create a GLib Main Loop and set it to run */
  data.main_loop = g_main_loop_new (NULL, FALSE);
  g_main_loop_run (data.main_loop);
 
  /* Free resources */
  gst_element_set_state (data.pipeline, GST_STATE_NULL);
  gst_object_unref (data.pipeline);
  return 0;
}

[Sebastian Dröge-3]

On Do, 2016-03-24 at 17:43 -0700, gyunaev wrote:
> Good morning,
>
> I've set up appsrc with format GST_FORMAT_BYTES, use push signal and set the
> buffer position via buffer->offset and buffer->offset_end. It works fine,
> but the maximum duration is reported.
>
> However if I also set the "size" property of appsrc, querying the duration
> fails - gst_element_query_duration() returns 0.

In your application you query position/duration in TIME format, but
appsrc can only answer duration queries in BYTES format. If you use
that, it should reply with the value you set for size.

It might make sense to also add a duration property to appsrc that
would be used for replying to the duration query in TIME format though.
If you need that, please file a bug here
  https://bugzilla.gnome.org/enter_bug.cgi?product=GStreamer

It's easy to add :)

--
Sebastian Dröge, Centricular Ltd · http://www.centricular.com


_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[gyunaev]

Thank you Sebastian. Is there a reason appsrc cannot auto-calculate duration in TIME units in this case? It has the format, caps and size, so should be trivial, right?

[Sebastian Dröge-3]

On Sa, 2016-03-26 at 06:27 -0700, gyunaev wrote:
> Thank you Sebastian. Is there a reason appsrc cannot auto-calculate duration
> in TIME units in this case? It has the format, caps and size, so should be
> trivial, right?

What's the duration of a 30MB h264 stream with 30fps? :)

Unfortunately it's not that simple for most formats, and the
information has to provided via other means.

--
Sebastian Dröge, Centricular Ltd · http://www.centricular.com


_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[gyunaev]

Indeed, it is not possible in every case. But when appsrc is serving video/x-raw or audio/x-raw, as it is the case with the tutorials, this should be fairly simple calculation.

And of course another issue here, as you pointed above, that even if I parse that h264 myself, know the duration and feed the appsrc with video/h264, there is no way to pass the duration to GStreamer.

[gyunaev]

This might also be somehow connected with another issue - appsrc in this configuration does not respond to gst_event_new_seek() - the seek callback is simply not called. I use:

    GstEvent * seek_event = gst_event_new_seek( m_tempoRate,
                                                GST_FORMAT_TIME,
                                                (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE),
                                                GST_SEEK_TYPE_SET,
                                                pos * GST_MSECOND,
                                                GST_SEEK_TYPE_NONE,
                                                0 );

but even when I change it to GST_FORMAT_BYTES, nothing happens. Pipeline generates no errors, but appsrc's seek_cb is not called, so no seek is performed. While I can seek directly in appsrc, this doesn't let me to change tempo, and thus is not suitable for my purpose.

I'm now thinking that I'm probably expecting too much from appsrc, and should be writing a plugin instead, am I right?

[Nicolas Dufresne-4]

Le samedi 26 mars 2016 à 15:41 -0700, gyunaev a écrit :

> This might also be somehow connected with another issue - appsrc in
> this
> configuration does not respond to gst_event_new_seek() - the seek
> callback
> is simply not called. I use:
>
>     GstEvent * seek_event = gst_event_new_seek( m_tempoRate,
>                                                 GST_FORMAT_TIME,
>                                                
> (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE),
>                                                 GST_SEEK_TYPE_SET,
>                                                 pos * GST_MSECOND,
>                                                 GST_SEEK_TYPE_NONE,
>                                                 0 );
>
> but even when I change it to GST_FORMAT_BYTES, nothing happens.
> Pipeline
> generates no errors, but appsrc's seek_cb is not called, so no seek
> is
> performed. While I can seek directly in appsrc, this doesn't let me
> to
> change tempo, and thus is not suitable for my purpose.

Have you set the stream type to SEEKABLE or RANDOM_ACCESS ? If you use
BYTE format, do you have a demuxer or a parser that will handle the
time to byte conversion ?

>
> I'm now thinking that I'm probably expecting too much from appsrc,
> and
> should be writing a plugin instead, am I right?
>
>
>
> --
> View this message in context: http://gstreamer-devel.966125.n4.nabble
> .com/appsrc-with-GST-FORMAT-BYTES-cannot-query-duration-
> tp4676539p4676575.html
> Sent from the GStreamer-devel mailing list archive at Nabble.com.
> _______________________________________________
> gstreamer-devel mailing list
> [hidden email]
> https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[gyunaev]

> Have you set the stream type to SEEKABLE or RANDOM_ACCESS ?

Yes, it is set to SEEKABLE. It does receive one seek in the seek-callback when I start playing it, but this seek is not caused by me; probably internal in GStreamer.

> If you use BYTE format, do you have a demuxer or a parser that will handle the time to byte conversion ?

No, I do not have anything specific set, as my appsrc generates audio/x-raw with preset caps and is linked directly with audioconverter, so there is no place for demuxer, and no need for decodebin.

Wouldn't the pipeline just call my seek callback and let me deal with it?

[Nicolas Dufresne-4]

Le samedi 26 mars 2016 à 17:00 -0700, gyunaev a écrit :
> No, I do not have anything specific set, as my appsrc generates
> audio/x-raw
> with preset caps and is linked directly with audioconverter, so there
> is no
> place for demuxer, and no need for decodebin. 
>
> Wouldn't the pipeline just call my seek callback and let me deal with
> it?

Ok, then your choice of BYTE format source along with TIME format seek
is questionable isn't it ? GStreamer won't guess which byte offset
matches the time you want to seek to. Try using TIME format and do the
conversion yourself. Remember that appsrc is format agnostic, so it
does not know that you have raw audio and that this conversion is near
to trivial.

Nicolas
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[gyunaev]

> Ok, then your choice of BYTE format source along with TIME format seek is questionable isn't it ?

It is, although I thought GStreamer would be able to do the conversion itself; for example, it guesses stream position properly.

Still I tried to use BYTES in seek event, and it didn't produce any visible changes; the callback is still not called.

Is apprsrc really format-agnostic, meaning it ignores the caps set? I thought the whole purpose of caps is to let the pipeline know about the format produced by appsrc, so it could use this to convert time, duration, etc. But this doesn't seem to be the case.

[Nicolas Dufresne-4]

Le samedi 26 mars 2016 à 17:24 -0700, gyunaev a écrit :
> Is apprsrc really format-agnostic, meaning it ignores the caps set? I
> thought the whole purpose of caps is to let the pipeline know about
> the
> format produced by appsrc, so it could use this to convert time,
> duration,
> etc. But this doesn't seem to be the case.

It is, it cannot (and does not) convert bytes to time and vis-versa.
The caps mechanism is also format agnostic, as it can be used by code
that does not know the meaning of the caps (code like appsrc).

Note, a way to get some conversion happening is to use RANDOM_ACCESS
mode instead. Assuming you have an audio sink that support this, in
RANDOM_ACCESS, the audio sink will operate in pull mode. The audio sink
know how to convert time to bytes for seeking. This feature of audio
sink is rarely used though, so don't build too much hope. You are
likely better using TIME format, and doing the conversion yourself.

Nicolas
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[gyunaev]

Thanks to Nicolas and Sebastian, changing the appsrc format to GST_FORMAT_TIME fixed the seeking issue.

For those having a similar issue and fixing it like that, note that:

1. You must timestamp the buffers instead of using offset and offset_end; and

2. Your seek callback now retrieves a time position, not the offset - you need to convert it to file offset yourself before calling your seek implementation.