[Android] [Appsink] [Appsrc] Strange behavior when displaying the receiving raw data via appsrc

[sofien rahmouni]

Hello,
i have an issue of strange behavior when display the received raw data h264
is half of the screen is grey.

The issue only reproduce on internet network (in local network is work
perfectely)

On my code i have two pipeline:

1 - Pipeline:  Appsink  which take the ahcsrc as a source video  and send it
as a raw by a UdpSocket on java

/ Sample Send data APPSINK
 data->pipeline = gst_parse_launch(
         "ahcsrc device=1 ! video/x-raw,format=(string)NV21 ! tee name=t t.
! queue ! autovideosink sync=false t. ! queue ! video/x-raw,  width=320,
height=240 ! videoconvert ! x264enc bitrate=500 speed-preset=superfast
tune=zerolatency ! rtph264pay mtu=1024 ! appsink
name=callback_read_buffer_sink emit-signals=true",
         &error);
 1.1 - Callback de l'appsink :

 GstElement *testsink = NULL;
 /* we use appsink in push mode, it sends us a signal when data is available
* and we pull out the data in the signal callback. */
 testsink = gst_bin_get_by_name(GST_BIN(data->pipeline),
"callback_read_buffer_sink");

 if (testsink == NULL) {
     g_print("appsink is NULL\n");
 }
 g_signal_connect(testsink, "new-sample", G_CALLBACK(cb_new_sample), NULL);
 gst_object_unref(testsink);

 if (error) {
     gchar *message =
             g_strdup_printf("Unable to build pipeline: %s",
error->message);
     g_clear_error(&error);
     set_ui_message(message, data);
     g_free(message);
     return NULL;
 }

2- Pipeline: Appsrc which receive a data raw video h264 and display it via
autovideosink

// Sample Receive data APPSRC
data->pipeline = gst_parse_launch(
        "appsrc name=video_app_source is-live=true !
application/x-rtp,media=video,clock-rate=90000,encoding-name=H264,payload=96
! rtpjitterbuffer latency=10 ! rtph264depay ! h264parse ! avdec_h264 !
videoconvert ! autovideosink",
        &error);
   2.1 - Callback de l'appsrc :
static void gst_native_receive_video_data(JNIEnv *env, jobject thiz,
jbyteArray array) {
   
    jbyte *temp = (*env)->GetByteArrayElements(env, array, NULL);
    jsize size = (*env)->GetArrayLength(env, array);

    GstBuffer *buffer = gst_buffer_new_allocate(NULL, size, NULL);
    gst_buffer_fill(buffer, 0, temp, size);

    GstElement *element =
gst_bin_get_by_name(GST_BIN(pCustomData->pipeline), "video_app_source");

    if (basetimestamp == 0) {

        if (appsrc) {
            element = gst_object_ref(appsrc);
        } else {
            fprintf(stdout, "%s:%d %s\n", __func__, __LINE__, "manually
setting time: exit");
            element = NULL;
            return;
        }

        GST_OBJECT_LOCK (element);
       
        if (GST_ELEMENT_CLOCK (element)) {
            GstClockTime now;
            GstClockTime base_time;
            now = gst_clock_get_time (GST_ELEMENT_CLOCK (element));
            base_time = GST_ELEMENT_CAST (element)->base_time;

            basetimestamp = now - base_time;
            GST_BUFFER_TIMESTAMP (buffer) = basetimestamp;
        }

        GST_OBJECT_UNLOCK (element);
        gst_object_unref (element);

        gst_app_src_push_buffer (GST_APP_SRC(element), buffer);
    } else {
        gst_app_src_push_buffer(appsrc, buffer);
    }

    (*env)->ReleaseByteArrayElements(env, array, temp, JNI_ABORT);
}

P.S:
Here is attached screenshot of the strange behavior of the streaming

<http://gstreamer-devel.966125.n4.nabble.com/file/t379283/Screenshot_20200302-165235%5B1%5D.png>



--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[Nicolas Dufresne-5]

Le mar. 3 mars 2020 05 h 30, sofien rahmouni <[hidden email]> a écrit :

Hello,
i have an issue of strange behavior when display the received raw data h264
is half of the screen is grey.

The issue only reproduce on internet network (in local network is work
perfectely)

On my code i have two pipeline:

1 - Pipeline:  Appsink  which take the ahcsrc as a source video  and send it
as a raw by a UdpSocket on java

/ Sample Send data APPSINK
 data->pipeline = gst_parse_launch(
         "ahcsrc device=1 ! video/x-raw,format=(string)NV21 ! tee name=t t.
! queue ! autovideosink sync=false t. ! queue ! video/x-raw,  width=320,
height=240 ! videoconvert ! x264enc bitrate=500 speed-preset=superfast
tune=zerolatency ! rtph264pay mtu=1024 ! appsink
name=callback_read_buffer_sink emit-signals=true",
         &error);
 1.1 - Callback de l'appsink :

 GstElement *testsink = NULL;
 /* we use appsink in push mode, it sends us a signal when data is available
* and we pull out the data in the signal callback. */
 testsink = gst_bin_get_by_name(GST_BIN(data->pipeline),
"callback_read_buffer_sink");

 if (testsink == NULL) {
     g_print("appsink is NULL\n");
 }
 g_signal_connect(testsink, "new-sample", G_CALLBACK(cb_new_sample), NULL);
 gst_object_unref(testsink);

 if (error) {
     gchar *message =
             g_strdup_printf("Unable to build pipeline: %s",
error->message);
     g_clear_error(&error);
     set_ui_message(message, data);
     g_free(message);
     return NULL;
 }

2- Pipeline: Appsrc which receive a data raw video h264 and display it via
autovideosink

// Sample Receive data APPSRC
data->pipeline = gst_parse_launch(
        "appsrc name=video_app_source is-live=true !
application/x-rtp,media=video,clock-rate=90000,encoding-name=H264,payload=96
! rtpjitterbuffer latency=10 ! rtph264depay ! h264parse ! avdec_h264 !
videoconvert ! autovideosink",
        &error);
   2.1 - Callback de l'appsrc :
static void gst_native_receive_video_data(JNIEnv *env, jobject thiz,
jbyteArray array) {

    jbyte *temp = (*env)->GetByteArrayElements(env, array, NULL);
    jsize size = (*env)->GetArrayLength(env, array);

    GstBuffer *buffer = gst_buffer_new_allocate(NULL, size, NULL);
    gst_buffer_fill(buffer, 0, temp, size);

    GstElement *element =
gst_bin_get_by_name(GST_BIN(pCustomData->pipeline), "video_app_source");

    if (basetimestamp == 0) {

        if (appsrc) {
            element = gst_object_ref(appsrc);
        } else {
            fprintf(stdout, "%s:%d %s\n", __func__, __LINE__, "manually
setting time: exit");
            element = NULL;
            return;
        }

        GST_OBJECT_LOCK (element);

        if (GST_ELEMENT_CLOCK (element)) {
            GstClockTime now;
            GstClockTime base_time;
            now = gst_clock_get_time (GST_ELEMENT_CLOCK (element));
            base_time = GST_ELEMENT_CAST (element)->base_time;

            basetimestamp = now - base_time;
            GST_BUFFER_TIMESTAMP (buffer) = basetimestamp;
        }

        GST_OBJECT_UNLOCK (element);
        gst_object_unref (element);

        gst_app_src_push_buffer (GST_APP_SRC(element), buffer);
    } else {
        gst_app_src_push_buffer(appsrc, buffer);
    }

    (*env)->ReleaseByteArrayElements(env, array, temp, JNI_ABORT);
}

P.S:
Here is attached screenshot of the strange behavior of the streaming

<http://gstreamer-devel.966125.n4.nabble.com/file/t379283/Screenshot_20200302-165235%5B1%5D.png>

Looks like a key frame was lost or never seen. Consider configuring a key frame interval on your encoder. Property key-int-max iirc.

--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[sofien rahmouni]

Thank you Nicolas for your response.

I tried to add the key-int-max property to the sender pipeline but it
does'nt fix my issue :/ .

ahcsrc device=1 ! video/x-raw,format=(string)NV21 ! tee name=t t. ! queue !
autovideosink sync=false t. ! queue ! video/x-raw,  width=320, height=240 !
videoconvert ! x264enc bitrate=500 speed-preset=superfast tune=zerolatency
byte-stream=true *key-int-max=30* ! rtph264pay mtu=1024 ! appsink
name=callback_read_buffer_sink emit-signals=true

In other side in the log, i see an error while decoding :

2020-03-03 15:50:00.383 18602-18794/com.test.demo
W/GStreamer+rtpjitterbuffer: 0:00:11.812912739 0x7dd6841cf0
../gst/rtpmanager/rtpjitterbuffer.c:749:rtp_jitter_buffer_calculate_pts
backward timestamps at server, schedule resync
2020-03-03 15:50:00.384 18602-18794/com.company.demo
W/GStreamer+rtpjitterbuffer: 0:00:11.813824406 0x7dd6841cf0
../gst/rtpmanager/rtpjitterbuffer.c:749:rtp_jitter_buffer_calculate_pts
backward timestamps at server, schedule resync
2020-03-03 15:50:00.436 18602-18793/com.company.demo E/GStreamer+libav:
0:00:11.865631911 0x7dd6841ca0 :0: top block unavailable for requested intra
mode
2020-03-03 15:50:00.436 18602-18802/com.company.demo E/GStreamer+libav:
0:00:11.865641078 0x7d882f98c0 :0: top block unavailable for requested intra
mode -1
2020-03-03 15:50:00.436 18602-18801/com.company.demo E/GStreamer+libav:
0:00:11.865628265 0x7d882f9e70 :0: top block unavailable for requested intra
mode -1
2020-03-03 15:50:00.436 18602-18802/com.company.demo E/GStreamer+libav:
0:00:11.865743942 0x7d882f98c0 :0: error while decoding MB 13 10, bytestream
1469
2020-03-03 15:50:00.436 18602-18793/com.company.demo E/GStreamer+libav:
0:00:11.865744046 0x7dd6841ca0 :0: error while decoding MB 10 0, bytestream
1930
2020-03-03 15:50:00.436 18602-18801/com.company.demo E/GStreamer+libav:
0:00:11.865745296 0x7d882f9e70 :0: error while decoding MB 8 5, bytestream
1676

i think there is a relation between the managing of the buffer
*rtpjitterbuffer *and the issue of decoding

Any help will be appreciated,

Thanks in advance



--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel

[sofien rahmouni]

Hello Nicolas,
I fixed my issue , it's was about the mtu value not setted in the rtp
payload *rtph264pay *:
// Sample Send data APPSINK
            data->pipeline = gst_parse_launch(
                    "ahcsrc device=1 ! video/x-raw,format=(string)NV21 ! tee
name=t t. ! queue ! autovideosink sync=false t. ! queue ! video/x-raw,
width=320, height=240 ! videoconvert ! x264enc bitrate=500 key-int-max=1 !
rtph264pay mtu=500 ! appsink name=callback_read_buffer_sink
emit-signals=true",
                    &error);

and the passed value from java

                        DatagramPacket dp = new DatagramPacket(msg,
msg.length);
                        ds.receive(dp);
                        //Send received frame data to native Gstreamer part
                       
nativeReceiveVideoData(Arrays.copyOf(dp.getData(),dp.getLength()));
 
but now i'm facing a new problem is video freez after some seconds, let's
see the log i found a warning about the timestamp and the rtpjitterbuffer
plugin.

here is my log :

03-09 15:29:49.363 22611-23151/com.valeo.voyagexr.demo
W/GStreamer+rtpjitterbuffer: 0:00:06.172550115 0x96c44a90
../gst/rtpmanager/rtpjitterbuffer.c:573:calculate_skew delta - skew:
0:00:04.165608667 too big, reset skew
03-09 15:30:03.743 22611-23151/com.valeo.voyagexr.demo
W/GStreamer+rtpjitterbuffer: 0:00:20.553730886 0x96c44a90
../gst/rtpmanager/rtpjitterbuffer.c:573:calculate_skew delta - skew:
0:00:01.019369667 too big, reset skew
03-09 15:30:03.888 22611-23148/com.valeo.voyagexr.demo
W/GStreamer+videodecoder: 0:00:20.701839732 0x96c44600
../gst-libs/gst/video/gstvideodecoder.c:2762:gst_video_decoder_prepare_finish_frame:<avdec_h264-0>
decreasing timestamp (0:00:17.161949336 < 0:00:17.622133705)
03-09 15:30:03.918 22611-23148/com.valeo.voyagexr.demo
W/GStreamer+videodecoder: 0:00:20.731212155 0x96c44600
../gst-libs/gst/video/gstvideodecoder.c:2762:gst_video_decoder_prepare_finish_frame:<avdec_h264-0>
decreasing timestamp (0:00:17.102651188 < 0:00:17.622133705)
03-09 15:30:04.008 22611-23148/com.valeo.voyagexr.demo
W/GStreamer+videodecoder: 0:00:20.819416962 0x96c44600
../gst-libs/gst/video/gstvideodecoder.c:2762:gst_video_decoder_prepare_finish_frame:<avdec_h264-0>
decreasing timestamp (0:00:16.986666736 < 0:00:17.622133705)


Could you please see the reason of this freeze

Thanks in advance of your help

Best Regards,
Sofien




--
Sent from: http://gstreamer-devel.966125.n4.nabble.com/
_______________________________________________
gstreamer-devel mailing list
[hidden email]
https://lists.freedesktop.org/mailman/listinfo/gstreamer-devel