Errors when using appsink/appsrc

package com.bumebox.core.red5.transcoder;

import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.util.Date;

import javax.imageio.ImageIO;

import org.freedesktop.gstreamer.Buffer;
import org.freedesktop.gstreamer.Bus;
import org.freedesktop.gstreamer.Caps;
import org.freedesktop.gstreamer.Element;
import org.freedesktop.gstreamer.ElementFactory;
import org.freedesktop.gstreamer.Gst;
import org.freedesktop.gstreamer.GstObject;
import org.freedesktop.gstreamer.Pad;
import org.freedesktop.gstreamer.Pipeline;
import org.freedesktop.gstreamer.Sample;
import org.freedesktop.gstreamer.State;
import org.freedesktop.gstreamer.Structure;
import org.freedesktop.gstreamer.elements.AppSink;
import org.freedesktop.gstreamer.elements.AppSrc;

import com.bumebox.core.red5.EnvConfig;

public class GStreamerOverlay {

	private boolean isFirstRead;

	private long initialDelayMs = 4000;

	private long overlayDurationMs = 5000;

	private long startOverlayMs;

	private boolean showOverlay = false;

	private BufferedImage waterMarkImage;

	public void startOverlay() throws Exception {
		String waterMarkFileName = EnvConfig.getFilePath("top_overlay.png");
		waterMarkImage = ImageIO.read(new File(waterMarkFileName));
		boolean enableAppSink = true;
		Gst.init("Test", new String[] { "--gst-debug-level=4" });
		// Gst.init("Test1", new String[] { });
		String srcLocation = "src/test/resources/video/iphone-video.MOV";
		// String srcLocation = "rtmp://192.168.1.6:1935/livetest/yyy";
		String rtmpSink = "rtmp://localhost:1935/livetest/yyy live=1";

		final Pipeline pipe = new Pipeline("First pipeline");
		Element source = ElementFactory.make("filesrc", "filesink");
		// Element source = ElementFactory.make("rtmpsrc", "filesink");
		Element demux = ElementFactory.make("qtdemux", "t");
		final Element h264decode = ElementFactory.make("avdec_h264", "h264");
		Element queue1 = ElementFactory.make("queue", "queue1");
		Element rgbConverter = ElementFactory.make("videoconvert",
				"rgbconverter");
		Element rgbCapsFilter = ElementFactory.make("capsfilter", "rgbcaps");
		String rgbCapsStr = "video/x-raw,format=RGB,width=1280,height=720";
		rgbCapsFilter.setCaps(new Caps(rgbCapsStr));

		final Element h264Converter = ElementFactory.make("videoconvert",
				"h264converter");
		Element h264CapsFilter = ElementFactory.make("capsfilter", "h264caps");
		String h264CapsStr = "video/x-raw,format=I420,width=1280,height=720";
		h264CapsFilter.setCaps(new Caps(h264CapsStr));
		Element queue2 = ElementFactory.make("queue", "queue2");
		Element x264enc = ElementFactory.make("x264enc", "x264");
		x264enc.set("tune", 0x00000004);
		// Element queue3 = ElementFactory.make("queue", "queue3");

		Element flvMux = ElementFactory.make("flvmux", "mux");
		final Element aacparse = ElementFactory.make("aacparse", "aac");
		Element queue4 = ElementFactory.make("queue", "queue4");
		Element rtmpsink = ElementFactory.make("rtmpsink", "rtmpSink");

		source.set("location", srcLocation);
		rtmpsink.set("location", rtmpSink);

		pipe.addMany(source, demux, h264decode, queue1, rgbConverter,
				rgbCapsFilter, h264Converter, h264CapsFilter, queue2, x264enc,
				flvMux, aacparse, queue4, rtmpsink);

		source.link(demux);

		h264decode.link(queue1);
		queue1.link(rgbConverter);
		rgbConverter.link(rgbCapsFilter);

		AppSink appSink = new AppSink("appSink");
		appSink.set("emit-signals", true);
		// appSink.set("sync", true);
		// appSink.set("sync", false);
		/*
		 * StringBuilder appSinkCaps = new StringBuilder("video/x-raw, "); //
		 * JNA creates ByteBuffer using native byte order, set masks according
		 * to that. if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
		 * appSinkCaps.append("format=BGRx"); } else {
		 * appSinkCaps.append("format=xRGB"); }
		 * appSinkCaps.append(",width=1280,height=720");
		 */
		appSink.setCaps(new Caps(rgbCapsStr));

		// Element tee = ElementFactory.make("tee", "split");
		// pipe.add(tee);
		final AppSrc appSrc = new AppSrc("appsrc") {
		};
		appSrc.setCaps(new Caps(rgbCapsStr));
		appSrc.set("is-live", true);
		appSrc.set("format", 3);
		appSrc.set("do-timestamp", true);
		appSrc.set("block", true);

		if (enableAppSink) {

			pipe.add(appSink);
			pipe.add(appSrc);
			// Element tQ1 = ElementFactory.make("queue", "tq1");
			// Element tQ2 = ElementFactory.make("queue", "tq2");
			// pipe.add(tQ1);
			// pipe.add(tQ2);
			// rgbCapsFilter.link(tee);

			// tee.link(tQ1);
			// tee.link(tQ2);

			// tQ1.link(h264Converter);
			// tQ2.link(appSink);
			// tQ1.link(appSink);

			rgbCapsFilter.link(appSink);
			// tQ1.link(appSink);
			appSrc.link(h264Converter);
			/*
			 * Bus.ERROR errorSignal = new Bus.ERROR() {
			 * 
			 * @Override public void errorMessage(GstObject source, int code,
			 * String message) { System.out.println("AppSrc error source:" +
			 * source + ", code:" + code + ", msg:" + message); } };
			 */
			// appSrc.getBus().connect(errorSignal);
			// appSrc.getBus().disconnect(errorSignal);
		} else {
			rgbCapsFilter.link(h264Converter);
		}
		h264Converter.link(h264CapsFilter);
		h264CapsFilter.link(queue2);
		queue2.link(x264enc);
		// x264enc.link(queue3);

		aacparse.link(queue4);
		// queue3.link(flvMux);
		x264enc.link(flvMux);
		queue4.link(flvMux);
		flvMux.link(rtmpsink);
		// pipe.debugToDotFile(5, "src/test/resources/temp.txt");
		demux.connect(new Element.PAD_ADDED() {

			@Override
			public void padAdded(Element ele, Pad pad) {
				System.out.println("Ele :" + ele + ", pad:" + pad);
				if (pad.getName().startsWith("audio")) {
					boolean success = Element.linkPads(ele, pad.getName(),
							aacparse, "sink");
					System.out.println("Audio link success :" + success);
				} else if (pad.getName().startsWith("video")) {
					boolean success = Element.linkPads(ele, pad.getName(),
							h264decode, "sink");
					System.out.println("Video link success :" + success);
				}
			}
		});

		appSink.connect(new AppSink.NEW_SAMPLE() {

			@Override
			public void newBuffer(AppSink appsink) {
				Sample sample = appsink.pullSample();
				Structure capsStruct = sample.getCaps().getStructure(0);
				int w = capsStruct.getInteger("width");
				int h = capsStruct.getInteger("height");
				String format = capsStruct.getString("format");
				Buffer buffer = sample.getBuffer();
				// ByteBuffer bBuffer = buffer.map(false);
				// byte[] arr = new byte[bBuffer.remaining()];
				// bBuffer.get(arr);
				// createImageFromBytes(arr);

				// System.out.println("Sample caps : [w=" + w + ",h=" + h +
				// ", format="
				// + format + "]");
				if (buffer != null) {
					appSrc.pushBuffer(buffer);
				}
				// buffer.unmap();
			}
		});

		/*
		 * appSink.connect(new AppSink.EOS() {
		 * 
		 * @Override public void eos(AppSink appsink) {
		 * System.out.println("EOS on " + appsink); } });
		 */

		appSink.connect(new AppSink.NEW_PREROLL() {

			@Override
			public void newPreroll(AppSink appsink) {
				System.out.println("NewPreroll on " + appsink);
				Sample sample = appsink.pullPreroll();
				Structure capsStruct = sample.getCaps().getStructure(0);
				int w = capsStruct.getInteger("width");
				int h = capsStruct.getInteger("height");
				String format = capsStruct.getString("format");
				Buffer buffer = sample.getBuffer();
				System.out.println("On PreRoll Sample caps : [w=" + w + ",h="
						+ h + ", format=" + format + "]");
				// ByteBuffer bBuffer = buffer.map(false);
				// byte[] arr = new byte[bBuffer.remaining()];
				// bBuffer.get(arr);
				// createImageFromBytes(arr);
				// Pad pad = h264Converter.getPads().get(0);
				// System.out.println(pad);
				// h264Converter.emit("push-buffer", buffer);;
				if (buffer != null) {
					appSrc.pushBuffer(buffer);
					System.out.println("Push buffer to appSrc in preroll");
				} else {
					System.out
							.println("Push buffer to appSrc in preroll is null");
				}
				// buffer.unmap();

			}
		});

		/*
		 * appSrc.connect(new AppSrc.PUSH_BUFFER() {
		 * 
		 * @Override public FlowReturn pushBuffer(AppSrc ele, Buffer buffer) {
		 * System.out.println("AppSrc pushbuffer:" + buffer); return
		 * FlowReturn.OK; } });
		 */

		/*
		 * appSrc.getBus().connect(new Bus.EOS() {
		 * 
		 * @Override public void endOfStream(GstObject obj) {
		 * System.out.println("AppSrc EOS:" + obj); } });
		 */

		/*
		 * appSrc.getBus().connect(new Bus.MESSAGE() {
		 * 
		 * @Override public void busMessage(Bus bus, Message msg) {
		 * System.out.println("AppSrc bus:" + bus + ", msg:" + msg);
		 * //h264Converter.postMessage(msg); } });
		 */

		System.out.println(pipe);
		pipe.play();
		pipe.getBus().connect(new Bus.EOS() {

			@Override
			public void endOfStream(GstObject gst) {
				System.out.println("Reached EOS:" + gst);
				Gst.quit();
			}
		});
		Gst.main();
		pipe.setState(State.NULL);
		Gst.deinit();
	}

	private BufferedImage createImageFromBytes(byte[] imageData) {
		BufferedImage image = null;
		if (imageData != null && imageData.length > 0) {
			ByteArrayInputStream bais = new ByteArrayInputStream(imageData);
			try {
				image = ImageIO.read(bais);
				long current = System.currentTimeMillis();
				if (!isFirstRead) {
					startOverlayMs = current + initialDelayMs;
					isFirstRead = true;
				}
				setOverlayState(current);
				if (showOverlay) {
					Graphics2D g = image.createGraphics();
					g.drawImage(waterMarkImage, 0, 0, null);
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		return image;
	}

	private void setOverlayState(long current) {
		if ((startOverlayMs - current) <= 0) {
			showOverlay = !showOverlay;
			startOverlayMs = current + overlayDurationMs;
			if (showOverlay) {
				System.out.println("Overlay started at :" + new Date(current));
			} else {
				System.out.println("Overlay stopped at :" + new Date(current));
			}
		}
	}

	public static void main(String[] args) {
		System.setProperty("jna.library.path",
				"/Library/Frameworks/GStreamer.framework/Versions/Current/lib");
		GStreamerOverlay overlay = new GStreamerOverlay();
		try {
			overlay.startOverlay();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
}

Errors when using appsink/appsrc

Errors when using appsink/appsrc

CODE

LOG

Re: Errors when using appsink/appsrc

Re: Errors when using appsink/appsrc

Re: Errors when using appsink/appsrc