diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | tests/examples/Makefile.am | 6 | ||||
| -rw-r--r-- | tests/examples/playout.c | 1103 |
3 files changed, 1110 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore index e69ee6471..aac56a80f 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ gst*orc.h /tests/examples/uvch264/test-uvch264 /tests/examples/mpegts/tsparser /tests/examples/opencv/gsthanddetect_test +/tests/examples/playout Build *.user diff --git a/tests/examples/Makefile.am b/tests/examples/Makefile.am index 6b752b9bc..009ff73f4 100644 --- a/tests/examples/Makefile.am +++ b/tests/examples/Makefile.am @@ -40,6 +40,12 @@ else GTK3_DIR= endif +noinst_PROGRAMS = playout + +playout_SOURCES = playout.c +playout_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) $(GST_CFLAGS) +playout_LDADD = $(GST_PLUGINS_BASE_LIBS) -lgstvideo-$(GST_API_VERSION) $(GST_LIBS) + SUBDIRS= mpegts $(DIRECTFB_DIR) $(GTK_EXAMPLES) $(OPENCV_EXAMPLES) $(GL_DIR) \ $(GTK3_DIR) $(AVSAMPLE_DIR) DIST_SUBDIRS= mpegts camerabin2 directfb mxf opencv uvch264 gl gtk avsamplesink diff --git a/tests/examples/playout.c b/tests/examples/playout.c new file mode 100644 index 000000000..29ae812c5 --- /dev/null +++ b/tests/examples/playout.c @@ -0,0 +1,1103 @@ +/* Copyright (C) 2015 Centricular Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include <gst/gst.h> +#include <gst/video/gstvideosink.h> + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +/* Change this to set the output resolution */ +#define OUTPUT_VIDEO_WIDTH 1280 +#define OUTPUT_VIDEO_HEIGHT 720 + +/* Video and audio caps outputted by the mixers */ +#define RAW_AUDIO_CAPS_STR "audio/x-raw, format=(string)S16LE, " \ +"layout=(string)interleaved, rate=(int)44100, channels=(int)2, " \ +"channel-mask=(bitmask)0x03" + +#define RAW_VIDEO_CAPS_STR "video/x-raw, width=(int)" STR(OUTPUT_VIDEO_WIDTH) \ +", height=(int)" STR(OUTPUT_VIDEO_HEIGHT) ", framerate=(fraction)25/1, " \ +"format=I420, pixel-aspect-ratio=(fraction)1/1, " \ +"interlace-mode=(string)progressive" + +GST_DEBUG_CATEGORY_STATIC (playout); +#define GST_CAT_DEFAULT playout + +typedef enum +{ + PLAYOUT_APP_STATE_READY, /* Newly created */ + PLAYOUT_APP_STATE_PLAYING, /* Playing an item */ + PLAYOUT_APP_STATE_EOS /* Finished playing, all items EOS */ +} PlayoutAppState; + +typedef struct +{ + /* Application state */ + PlayoutAppState state; + + /* An array of PlayoutItems that will be played in sequence */ + GPtrArray *play_queue; + /* Index of the currently-playing item */ + gint play_queue_current; + /* Lock access to the play queue */ + GMutex play_queue_lock; + + GMainLoop *main_loop; + + /* Pipeline */ + GstElement *pipeline; + + /* Output */ + GstElement *video_mixer; + GstElement *video_sink; + GstVideoRectangle video_orect; /* w/h/x/y of the output */ + + GstElement *audio_mixer; + GstElement *audio_sink; + + /* The duration of all items that have been played in ns. + * Only updates when a new item is activated. */ + guint64 elapsed_duration; +} PlayoutApp; + +typedef enum +{ + PLAYOUT_ITEM_STATE_NEW, /* Newly created */ + PLAYOUT_ITEM_STATE_PREPARED, /* Prepared and ready to activate */ + PLAYOUT_ITEM_STATE_ACTIVATED, /* Activated */ + PLAYOUT_ITEM_STATE_FIRST_VBUFFER, /* First video buffer has gone through */ + PLAYOUT_ITEM_STATE_AGGREGATING, /* Audio & video buffers are aggregating */ + PLAYOUT_ITEM_STATE_EOS /* At least one pad is EOS */ +} PlayoutItemState; + +typedef struct +{ + PlayoutApp *app; + PlayoutItemState state; + + gchar *fn; + + GstElement *decoder; /* bin with uridecodebin + converters */ + + /* We just use the first audio stream and ignore the rest (if there is audio) */ + GstPad *audio_pad; /* decoder bin audio src ghostpad */ + GstPad *video_pad; /* decoder bin video src ghostpad */ + GstVideoRectangle video_irect; /* input w/h/x/y of the item */ + GstVideoRectangle video_orect; /* output w/h/x/y of the item */ + + /* When this item has finished preparing and all pads have been connected to + * mixers, the pads will be blocked till it's this item's turn to be played */ + gulong audio_pad_probe_block_id; + gulong video_pad_probe_block_id; + + /* The current running time of this item; updated with every audio buffer if + * this item has audio; otherwise it's updated withe very video buffer */ + guint64 running_time; +} PlayoutItem; + +static PlayoutApp *playout_app_new (void); +static void playout_app_free (PlayoutApp * app); +static PlayoutItem *playout_item_new (PlayoutApp * app, const gchar * fn); +static void playout_item_free (PlayoutItem * item); + +static void playout_app_add_item (PlayoutApp * app, const gchar * fn); +static gboolean playout_app_prepare_item (PlayoutItem * item); +static gboolean playout_app_activate_item (PlayoutItem * item); +static gboolean playout_app_activate_next_item (PlayoutApp * app); +static gboolean playout_app_activate_next_item_early (PlayoutApp * app); +static PlayoutItem *playout_app_get_current_item (PlayoutApp * app); +static gboolean playout_app_remove_item (PlayoutItem * item); + +static void +playout_app_add_audio_sink (PlayoutApp * app) +{ + GstElement *audio_resample, *audio_conv, *queue; + + /* audiomixer doesn't do conversion yet, so we don't need an output capsfilter + * for this branch. The output format is decided by the sink pads, which all + * have to have the same format. */ + app->audio_mixer = gst_element_factory_make ("audiomixer", "audio_mixer"); + audio_conv = gst_element_factory_make ("audioconvert", "mixer_audioconvert"); + audio_resample = gst_element_factory_make ("audioresample", + "audio_mixer_audioresample"); + queue = gst_element_factory_make ("queue", "asink_queue"); + app->audio_sink = gst_element_factory_make ("autoaudiosink", NULL); + g_object_set (app->audio_sink, "async-handling", TRUE, NULL); + gst_bin_add_many (GST_BIN (app->pipeline), app->audio_mixer, audio_conv, + audio_resample, queue, app->audio_sink, NULL); + gst_element_link_many (app->audio_mixer, audio_conv, audio_resample, + queue, app->audio_sink, NULL); + + if (!gst_element_sync_state_with_parent (app->audio_mixer) || + !gst_element_sync_state_with_parent (audio_conv) || + !gst_element_sync_state_with_parent (audio_resample) || + !gst_element_sync_state_with_parent (queue) || + !gst_element_sync_state_with_parent (app->audio_sink)) + GST_ERROR ("app: unable to sync audio mixer + sink state with pipeline"); +} + +static PlayoutApp * +playout_app_new (void) +{ + GstElement *video_capsfilter, *queue; + GstCaps *caps; + PlayoutApp *app; + + app = g_new0 (PlayoutApp, 1); + + app->state = PLAYOUT_APP_STATE_READY; + + app->play_queue = + g_ptr_array_new_with_free_func ((GDestroyNotify) playout_item_free); + app->play_queue_current = -1; + g_mutex_init (&app->play_queue_lock); + + app->main_loop = g_main_loop_new (NULL, FALSE); + + app->pipeline = gst_pipeline_new ("pipeline"); + + /* It's best to set a caps filter for the video output format */ + app->video_orect.w = OUTPUT_VIDEO_WIDTH; + app->video_orect.h = OUTPUT_VIDEO_HEIGHT; + app->video_orect.x = 0; + app->video_orect.y = 0; + app->video_mixer = gst_element_factory_make ("compositor", "video_mixer"); + /* Set the background as black; faster while compositing, and allows us to + * rescale videos with a different aspect ratio than the output in a way that + * adds black borders automatically */ + g_object_set (app->video_mixer, "background", 1, NULL); + queue = gst_element_factory_make ("queue", "vsink_queue"); + app->video_sink = gst_element_factory_make ("autovideosink", NULL); + g_object_set (app->video_sink, "async-handling", TRUE, NULL); + video_capsfilter = gst_element_factory_make ("capsfilter", + "video_mixer_capsfilter"); + caps = gst_caps_from_string (RAW_VIDEO_CAPS_STR); + g_object_set (video_capsfilter, "caps", caps, NULL); + gst_caps_unref (caps); + gst_bin_add_many (GST_BIN (app->pipeline), app->video_mixer, video_capsfilter, + queue, app->video_sink, NULL); + gst_element_link_many (app->video_mixer, video_capsfilter, queue, + app->video_sink, NULL); + + return app; +} + +static void +playout_app_free (PlayoutApp * app) +{ + GST_DEBUG ("Freeing app"); + g_ptr_array_unref (app->play_queue); + g_main_loop_unref (app->main_loop); + gst_element_set_state (app->pipeline, GST_STATE_NULL); + gst_object_unref (app->pipeline); + g_free (app); +} + +static void +playout_app_eos (GstBus * bus, GstMessage * msg, PlayoutApp * app) +{ + g_print ("All streams EOS, exiting...\n"); + g_main_loop_quit (app->main_loop); +} + +static PlayoutItem * +playout_item_new (PlayoutApp * app, const gchar * fn) +{ + PlayoutItem *item = g_new0 (PlayoutItem, 1); + + item->app = app; + item->state = PLAYOUT_ITEM_STATE_NEW; + item->fn = g_strdup (fn); + + return item; +} + +/* Unlink and release the pad */ +static gboolean +playout_remove_pad (GstPad * srcpad) +{ + GstPad *sinkpad; + GstElement *mixer; + + sinkpad = gst_pad_get_peer (srcpad); + if (!sinkpad) + return FALSE; + if (!gst_pad_unlink (srcpad, sinkpad)) + return FALSE; + mixer = gst_pad_get_parent_element (sinkpad); + gst_element_release_request_pad (mixer, sinkpad); + GST_DEBUG ("Released some pad"); + + gst_object_unref (sinkpad); + gst_object_unref (mixer); + return FALSE; +} + +static GstPadProbeReturn +playout_item_pad_probe_blocked (GstPad * srcpad, GstPadProbeInfo * info, + PlayoutItem * item) +{ + if (srcpad == item->audio_pad) { + item->audio_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info); + } else if (srcpad == item->video_pad) { + item->video_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info); + } else { + g_assert_not_reached (); + } + + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn +playout_item_pad_probe_pad_running_time (GstPad * srcpad, + GstPadProbeInfo * info, PlayoutItem * item) +{ + GstEvent *event; + GstBuffer *buffer; + guint64 running_time; + const GstSegment *segment; + + buffer = GST_PAD_PROBE_INFO_BUFFER (info); + event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0); + GST_TRACE ("%s: pad sticky event: %" GST_PTR_FORMAT, item->fn, event); + + if (event) { + gst_event_parse_segment (event, &segment); + gst_event_unref (event); + running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME, + GST_BUFFER_PTS (buffer)); + } else { + GST_WARNING ("%s: unable to parse event for segment; falling back to pts. " + "Output will probably have glitches.", item->fn); + running_time = GST_BUFFER_PTS (buffer); + } + + item->running_time = running_time + GST_BUFFER_DURATION (buffer); + GST_TRACE ("%s: running time is %" G_GUINT64_FORMAT ", duration is %" + G_GUINT64_FORMAT, item->fn, item->running_time, + GST_BUFFER_DURATION (buffer)); + + return GST_PAD_PROBE_PASS; +} + +static GstPadProbeReturn +playout_item_pad_probe_video_pad_eos_on_buffer (GstPad * srcpad, + GstPadProbeInfo * info, PlayoutItem * prev_item) +{ + PlayoutItem *current_item; + + current_item = playout_app_get_current_item (prev_item->app); + + if (!current_item) + return GST_PAD_PROBE_REMOVE; + + /* Step through the item's states as buffers pass through. The first buffer + * will be taken by the video_mixer, and kept till the audio running time + * matches the video buffer running time. When the second buffer gets through, + * we know that this pad has begun aggregating. */ + switch (current_item->state) { + case PLAYOUT_ITEM_STATE_NEW: + case PLAYOUT_ITEM_STATE_PREPARED: + GST_DEBUG ("%s: new/prepared", current_item->fn); + break; + case PLAYOUT_ITEM_STATE_ACTIVATED: + GST_DEBUG ("%s: activated -> first vbuffer", current_item->fn); + current_item->state = PLAYOUT_ITEM_STATE_FIRST_VBUFFER; + break; + case PLAYOUT_ITEM_STATE_FIRST_VBUFFER: + GST_DEBUG ("%s: first vbuffer -> aggregating", current_item->fn); + current_item->state = PLAYOUT_ITEM_STATE_AGGREGATING; + gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info)); + /* Item is aggregating, release the previous item's video pad */ + goto release; + break; + case PLAYOUT_ITEM_STATE_EOS: + return GST_PAD_PROBE_REMOVE; + default: + g_assert_not_reached (); + } + + return GST_PAD_PROBE_PASS; + +release: + { + playout_remove_pad (prev_item->video_pad); + GST_DEBUG ("%s: released video pad", prev_item->fn); + prev_item->video_pad = NULL; + + /* If there's no audio pad, or if the audio pad is already EOS, we can + * remove this item from the queue which will free it. Need to free the + * item from the main thread because it causes the item's decoder bin + * to be removed from the pipeline, which cannot be done in the + * streaming thread */ + if (prev_item->audio_pad == NULL) { + GST_DEBUG ("%s: queued item removal (last pad is video)", prev_item->fn); + g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item, + prev_item); + } + + /* Pad probe has already been removed above */ + return GST_PAD_PROBE_PASS; + } +} + +/* This is called on EOS for both item->audio_pad and item->video_pad + * + * FIXME: Add locking. Both pads could go EOS at the exact same time. */ +static GstPadProbeReturn +playout_item_pad_probe_event (GstPad * srcpad, GstPadProbeInfo * info, + PlayoutItem * item) +{ + GstEventType type; + gboolean ret = TRUE; + GstPadProbeReturn probe_ret = GST_PAD_PROBE_DROP; + + type = GST_EVENT_TYPE (GST_PAD_PROBE_INFO_DATA (info)); + if (type != GST_EVENT_EOS) + return GST_PAD_PROBE_PASS; + + /* We might get two EOSes on this pad if we send an artificial EOS. Remove + * the probe so this is only called once for each pad */ + gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info)); + + GST_DEBUG ("%s: recvd some EOS", item->fn); + + if (item->state != PLAYOUT_ITEM_STATE_EOS) { + /* We have more than one pad per item (video + audio item), and this is the + * first pad to go EOS or we have only one pad per item, and that pad has + * gone EOS. For the first case, the other pad might still have some buffers + * to output before going EOS, but we need to activate the next item and + * start outputting buffers from that immediately. */ + + /* Update the total elapsed duration from the item's current running time */ + item->app->elapsed_duration += item->running_time; + + GST_DEBUG ("%s: activating next item", item->fn); + /* Activate the next item if and only if this is the first pad to go EOS */ + ret = playout_app_activate_next_item (item->app); + if (!ret) { + GST_DEBUG ("%s: App is going EOS", item->fn); + item->state = PLAYOUT_ITEM_STATE_EOS; + item->app->state = PLAYOUT_APP_STATE_EOS; + /* If we couldn't activate the next item, pass the EOS event onward, + * ending the stream */ + probe_ret = GST_PAD_PROBE_PASS; + } + } + + g_assert (srcpad != NULL); + + if (srcpad == item->audio_pad) { + GST_DEBUG ("%s: audio pad was EOS", item->fn); + + if (item->app->state != PLAYOUT_APP_STATE_EOS) { + /* While activating the next item, we ensure that there's no offset mismatch + * which would cause audiomixer to output silence, so we can release the + * previous item's audio request pad here. We also unlink the audio pad; + * nothing else is needed from it */ + playout_remove_pad (srcpad); + GST_DEBUG ("%s: released audio pad", item->fn); + + /* If there's no video pad, or if the video pad is already EOS, we can + * remove this item from the queue which will free it. Need to free the + * item from the main thread because it causes the item's decoder bin + * to be removed from the pipeline, which cannot be done in the + * streaming thread */ + if (item->video_pad == NULL) { + GST_DEBUG ("%s: queued item removal (last pad is audio)", item->fn); + g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item, + item); + } + } else { + /* If this is the last pad on audio_mixer, let the EOS go through + * before unlinking/releasing the pad. This should happen within 500ms. */ + g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad); + GST_DEBUG ("%s: queued audio pad release", item->fn); + + if (item->video_pad == NULL) { + /* Unlike above, we need to wait till the pad is removed before removing + * the item from the app, so we queue it for 100ms afterwards */ + GST_DEBUG ("%s: queued last item removal (last pad is audio)", + item->fn); + g_timeout_add (600, (GSourceFunc) playout_app_remove_item, item); + } + } + item->audio_pad = NULL; + } else if (srcpad == item->video_pad) { + + GST_DEBUG ("%s: video pad was EOS", item->fn); + + if (item->audio_pad != NULL) + GST_WARNING ("%s: video pad went EOS before audio pad! " + "There will be audio/video glitches while switching.", item->fn); + + if (item->app->state != PLAYOUT_APP_STATE_EOS) { + PlayoutItem *next_item; + + next_item = playout_app_get_current_item (item->app); + GST_DEBUG ("%s: next item is %s, %i/%i", item->fn, next_item->fn, + next_item->state, PLAYOUT_ITEM_STATE_ACTIVATED); + + g_assert (next_item != NULL); + /* If there's another item being activated, release this video pad only + * when the next item's video pad starts being aggregated; that happens + * when this probe receives its 2nd buffer from the next item */ + gst_pad_add_probe (next_item->video_pad, GST_PAD_PROBE_TYPE_BUFFER, + (GstPadProbeCallback) playout_item_pad_probe_video_pad_eos_on_buffer, + item, NULL); + } else { + /* If this is the last pad on video_mixer, let the EOS go through + * before unlinking/releasing the pad. This should happen within 500ms. */ + g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad); + GST_DEBUG ("%s: queued video pad release", item->fn); + item->video_pad = NULL; + } + probe_ret = GST_PAD_PROBE_PASS; + } else { + g_assert_not_reached (); + } + + item->state = PLAYOUT_ITEM_STATE_EOS; + + /* NOTE: If the srcpad has been unlinked, the return value is useless */ + return probe_ret; +} + +/* On the "pad-added" signal of uridecodebin, add converters and connect to + * audio/video mixers */ +static void +playout_item_new_pad (GstElement * uridecodebin, GstPad * pad, + PlayoutItem * item) +{ + GstStructure *s; + GstCaps *caps; + GstPad *sinkpad, *srcpad; + GstElement *queue; + GstPadProbeType block_probe_type; + + caps = gst_pad_get_current_caps (pad); + s = gst_caps_get_structure (caps, 0); + GST_DEBUG ("%s: new pad: %p, caps: %s", item->fn, pad, + gst_structure_get_name (s)); + + if (gst_structure_has_name (s, "audio/x-raw")) { + if (item->audio_pad != NULL) + /* Ignore all audio pads after the first one */ + goto out; + goto audio; + } else if (gst_structure_has_name (s, "video/x-raw")) { + if (item->video_pad != NULL) + /* Ignore all video pads after the first one */ + goto out; + goto video; + } else { + goto out; + } + +audio: + { + GstCaps *wanted_caps; + GstElement *audioconvert, *audioresample, *capsfilter; + + /* Audio pad found; add audio mixer and audio sink to the pipeline. + * NOTE: If any items after this do not have an audio pad, the pipeline will + * mess up because the audio sink will not receive any data. */ + if (item->app->audio_sink == NULL) + playout_app_add_audio_sink (item->app); + + wanted_caps = gst_caps_from_string (RAW_AUDIO_CAPS_STR); + + if (!gst_caps_is_equal (caps, wanted_caps)) { + GST_DEBUG ("%s: converting audio caps", item->fn); + /* We need to convert the audio to the wanted format because + * audiomixer doesn't do format conversion */ + audioresample = gst_element_factory_make ("audioresample", NULL); + audioconvert = gst_element_factory_make ("audioconvert", NULL); + capsfilter = gst_element_factory_make ("capsfilter", NULL); + g_object_set (capsfilter, "caps", wanted_caps, NULL); + queue = gst_element_factory_make ("queue", NULL); + gst_bin_add_many (GST_BIN (item->decoder), audioresample, audioconvert, + capsfilter, queue, NULL); + + sinkpad = gst_element_get_static_pad (audioresample, "sink"); + gst_pad_link (pad, sinkpad); + gst_object_unref (sinkpad); + gst_element_link_many (audioresample, audioconvert, capsfilter, queue, + NULL); + srcpad = gst_element_get_static_pad (queue, "src"); + + if (!gst_element_sync_state_with_parent (audioresample) || + !gst_element_sync_state_with_parent (audioconvert) || + !gst_element_sync_state_with_parent (capsfilter) || + !gst_element_sync_state_with_parent (queue)) { + GST_ERROR ("%s: unable to sync audio converter state with decoder", + item->fn); + goto out; + } + } else { + queue = gst_element_factory_make ("queue", NULL); + gst_bin_add (GST_BIN (item->decoder), queue); + sinkpad = gst_element_get_static_pad (queue, "sink"); + gst_pad_link (pad, sinkpad); + gst_object_unref (sinkpad); + + srcpad = gst_element_get_static_pad (queue, "src"); + + if (!gst_element_sync_state_with_parent (queue)) { + GST_ERROR ("%s: unable to sync audio queue state with decoder", + item->fn); + goto out; + } + } + gst_caps_unref (wanted_caps); + + /* Convert the audioconvert src pad to a ghostpad on the bin */ + item->audio_pad = gst_ghost_pad_new (NULL, srcpad); + gst_pad_set_active (item->audio_pad, TRUE); + gst_element_add_pad (item->decoder, item->audio_pad); + gst_object_unref (srcpad); + + srcpad = item->audio_pad; + GST_DEBUG ("%s: created audio pad", item->fn); + goto done; + } + +video: + { + if (!gst_structure_get_int (s, "width", &item->video_irect.w) || + !gst_structure_get_int (s, "height", &item->video_irect.h)) + GST_WARNING ("%s: unable to set width/height from caps", item->fn); + item->video_irect.x = item->video_irect.y = 0; + + queue = gst_element_factory_make ("queue", "video-decoder-queue-%u"); + gst_bin_add (GST_BIN (item->decoder), queue); + + if (!gst_element_sync_state_with_parent (queue)) { + GST_ERROR ("%s: unable to sync video queue state with decoder", item->fn); + goto out; + } + + sinkpad = gst_element_get_static_pad (queue, "sink"); + gst_pad_link (pad, sinkpad); + gst_object_unref (sinkpad); + + /* Convert the queue src pad to a ghostpad on the bin */ + srcpad = gst_element_get_static_pad (queue, "src"); + item->video_pad = gst_ghost_pad_new (NULL, srcpad); + gst_pad_set_active (item->video_pad, TRUE); + gst_element_add_pad (item->decoder, item->video_pad); + gst_object_unref (srcpad); + + srcpad = item->video_pad; + GST_DEBUG ("%s: created video pad", item->fn); + goto done; + } + +done: + /* We let events and queries through */ + block_probe_type = GST_PAD_PROBE_TYPE_BLOCK | + GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST; + /* If the app is already playing an item, block everything except queries + * till we need to play this item */ + if (item->app->state != PLAYOUT_APP_STATE_READY) + gst_pad_add_probe (srcpad, block_probe_type, + (GstPadProbeCallback) playout_item_pad_probe_blocked, item, NULL); + /* Probe events for EOS */ + gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, + (GstPadProbeCallback) playout_item_pad_probe_event, item, NULL); + +out: + gst_caps_unref (caps); +} + +/* All pads on uridecodebin have finished being populated; the item has been + * prepared and is ready to be activated */ +static void +playout_item_no_more_pads (GstElement * uridecodebin, PlayoutItem * item) +{ + /* Set a buffer pad probe that constantly updates the item's + * elapsed_duration using the duration of each audio buffer */ + if (item->audio_pad) { + gst_pad_add_probe (item->audio_pad, GST_PAD_PROBE_TYPE_BUFFER, + (GstPadProbeCallback) playout_item_pad_probe_pad_running_time, + item, NULL); + } else if (item->video_pad) { + gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER, + (GstPadProbeCallback) playout_item_pad_probe_pad_running_time, + item, NULL); + } else { + GST_ERROR ("%s: no pads were generated! Can't continue playing!", item->fn); + return; + } + + item->state = PLAYOUT_ITEM_STATE_PREPARED; + GST_DEBUG ("%s: prepared", item->fn); + + if (item->app->state != PLAYOUT_APP_STATE_READY) + /* This item will be activated when the previous one is EOS */ + return; + + GST_DEBUG ("Application isn't already playing; activate the item and prepare" + " the next one"); + + playout_app_activate_item (item); + item->state = PLAYOUT_ITEM_STATE_ACTIVATED; + item->app->state = PLAYOUT_APP_STATE_PLAYING; + + if (item->app->play_queue->len > 1) + playout_app_prepare_item (g_ptr_array_index (item->app->play_queue, 1)); +} + +static GstElement * +playout_item_create_decoder (PlayoutItem * item) +{ + GstElement *bin, *dec; + GError *err = NULL; + gchar *uri; + + uri = gst_filename_to_uri (item->fn, &err); + if (err != NULL) { + GST_WARNING ("Could not convert '%s' to uri: %s", item->fn, err->message); + g_error_free (err); + return NULL; + } + + bin = gst_bin_new (NULL); + dec = gst_element_factory_make ("uridecodebin", NULL); + g_object_set (dec, "uri", uri, NULL); + g_free (uri); + + gst_bin_add (GST_BIN (bin), dec); + + g_signal_connect (dec, "pad-added", G_CALLBACK (playout_item_new_pad), item); + g_signal_connect (dec, "no-more-pads", G_CALLBACK (playout_item_no_more_pads), + item); + + return bin; +} + +static void +playout_item_free (PlayoutItem * item) +{ + GST_DEBUG ("Entering free"); + switch (gst_element_set_state (item->decoder, GST_STATE_NULL)) { + case GST_STATE_CHANGE_FAILURE: + GST_ERROR ("%s: Unable to change state to NULL", item->fn); + break; + case GST_STATE_CHANGE_SUCCESS: + GST_DEBUG ("%s: State change success", item->fn); + break; + default: + GST_DEBUG ("%s: Some async/no-preroll", item->fn); + } + + gst_bin_remove (GST_BIN (item->app->pipeline), item->decoder); + GST_DEBUG ("%s: bin removed", item->fn); + + g_free (item->fn); + g_free (item); + GST_DEBUG ("item freed"); +} + +static guint64 +playout_item_pad_get_segment_time (GstPad * srcpad) +{ + GstEvent *event; + const GstSegment *segment; + + event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0); + if (!event) + return 0; + gst_event_parse_segment (event, &segment); + gst_event_unref (event); + return segment->time; +} + +static void +playout_app_add_item (PlayoutApp * app, const gchar * fn) +{ + PlayoutItem *item; + + item = playout_item_new (app, fn); + + g_mutex_lock (&app->play_queue_lock); + g_ptr_array_add (app->play_queue, item); + g_mutex_unlock (&app->play_queue_lock); +} + +static gboolean +playout_app_remove_item (PlayoutItem * item) +{ + PlayoutApp *app; + GST_DEBUG ("%s: removing and freeing", item->fn); + + app = item->app; + + g_mutex_lock (&app->play_queue_lock); + g_ptr_array_remove (app->play_queue, item); + /* This item has been removed from the array, decrement the index */ + app->play_queue_current--; + g_mutex_unlock (&app->play_queue_lock); + + /* Don't call this again */ + return FALSE; +} + +static PlayoutItem * +playout_app_get_current_item (PlayoutApp * app) +{ + if (app->play_queue_current < 0 || + app->play_queue->len < (app->play_queue_current + 1)) + return NULL; + + return g_ptr_array_index (app->play_queue, app->play_queue_current); +} + +static gboolean +playout_app_prepare_item (PlayoutItem * item) +{ + PlayoutApp *app = item->app; + + if (item->decoder != NULL) + return TRUE; + + item->decoder = playout_item_create_decoder (item); + + if (item->decoder == NULL) + return FALSE; + + gst_bin_add (GST_BIN (app->pipeline), item->decoder); + + if (!gst_element_sync_state_with_parent (item->decoder)) { + GST_ERROR ("%s: unable to sync state with parent", item->fn); + return FALSE; + } + + GST_DEBUG ("%s: preparing", item->fn); + + /* All further processing is done in the "no-more-pads" callback of + * uridecodebin */ + return TRUE; +} + +/* Called exactly once for each item */ +static gboolean +playout_app_activate_item (PlayoutItem * item) +{ + GstPad *sinkpad; + guint64 segment_time; + PlayoutApp *app = item->app; + + if (item->state != PLAYOUT_ITEM_STATE_PREPARED) { + GST_ERROR ("Item %s is not ready to be activated!", item->fn); + return FALSE; + } + + if (!item->audio_pad && !item->video_pad) { + GST_ERROR ("Item %s has no pads! Can't activate it!", item->fn); + return FALSE; + } + + /* Hook up to mixers and remove the probes blocking the pads */ + if (item->audio_pad) { + GST_DEBUG ("%s: hooking up audio pad to mixer", item->fn); + sinkpad = gst_element_get_request_pad (app->audio_mixer, "sink_%u"); + gst_pad_link (item->audio_pad, sinkpad); + + segment_time = playout_item_pad_get_segment_time (item->audio_pad); + if (segment_time > 0) { + /* If the segment time is > 0, the new pad wants audiomixer to output audio + * silence for that duration. This will cause audio glitches, so we move + * the pad offset back by that amount and tell audiomixer to start mixing + * our buffers immediately. */ + GST_DEBUG ("%s: subtracting segment time %" G_GUINT64_FORMAT " from " + "elapsed duration before setting it as the pad offset", item->fn, + segment_time); + if (app->elapsed_duration > segment_time) + app->elapsed_duration -= segment_time; + else + app->elapsed_duration = 0; + } + + if (app->elapsed_duration > 0) { + GST_DEBUG ("%s: set audio pad offset to %" G_GUINT64_FORMAT "ms", + item->fn, app->elapsed_duration / GST_MSECOND); + gst_pad_set_offset (item->audio_pad, app->elapsed_duration); + } + + if (item->audio_pad_probe_block_id > 0) { + GST_DEBUG ("%s: removing audio pad block probe", item->fn); + gst_pad_remove_probe (item->audio_pad, item->audio_pad_probe_block_id); + } + gst_object_unref (sinkpad); + } + + if (item->video_pad) { + GST_DEBUG ("%s: hooking up video pad to mixer", item->fn); + sinkpad = gst_element_get_request_pad (app->video_mixer, "sink_%u"); + + /* Get new height/width/xpos/ypos such that the video scales up or down to + * fit within the output video size without any cropping */ + gst_video_sink_center_rect (item->video_irect, item->app->video_orect, + &item->video_orect, TRUE); + GST_DEBUG ("%s: w: %i, h: %i, x: %i, y: %i\n", item->fn, + item->video_orect.w, item->video_orect.h, item->video_orect.x, + item->video_orect.y); + g_object_set (sinkpad, "width", item->video_orect.w, "height", + item->video_orect.h, "xpos", item->video_orect.x, "ypos", + item->video_orect.y, NULL); + + /* If this is not the last item, on EOS, continue to aggregate using the + * last buffer till the pad is released */ + if (item->app->play_queue->len != (item->app->play_queue_current + 2)) + g_object_set (sinkpad, "ignore-eos", TRUE, NULL); + else + GST_DEBUG ("%s: last item, not setting ignore-eos", item->fn); + gst_pad_link (item->video_pad, sinkpad); + + if (app->elapsed_duration > 0) { + GST_DEBUG ("%s: set video pad offset to %" G_GUINT64_FORMAT "ms", + item->fn, app->elapsed_duration / GST_MSECOND); + gst_pad_set_offset (item->video_pad, app->elapsed_duration); + } + + if (item->video_pad_probe_block_id > 0) { + GST_DEBUG ("%s: removing video pad block probe", item->fn); + gst_pad_remove_probe (item->video_pad, item->video_pad_probe_block_id); + } + gst_object_unref (sinkpad); + } + + item->state = PLAYOUT_ITEM_STATE_ACTIVATED; + g_mutex_lock (&item->app->play_queue_lock); + item->app->play_queue_current++; + g_mutex_unlock (&item->app->play_queue_lock); + + GST_DEBUG ("%s: activated", item->fn); + + return TRUE; +} + +/* Activate the next item, and prepare the one after that for later activation */ +static gboolean +playout_app_activate_next_item (PlayoutApp * app) +{ + PlayoutItem *item; + gboolean ret; + + if (app->play_queue->len < (app->play_queue_current + 2)) { + g_print ("No more items to play\n"); + return FALSE; + } + + item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1); + ret = playout_app_activate_item (item); + if (!ret) { + /* Tell caller, who can then decide whether to skip or error out */ + GST_ERROR ("%s: unable to activate", item->fn); + return FALSE; + } + if (app->play_queue->len > (app->play_queue_current + 1)) { + item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1); + /* FIXME: What if this fails? Prepare the next one in the queue? */ + ret = playout_app_prepare_item (item); + if (!ret) + GST_ERROR ("%s: unable to prepare", item->fn); + } + return ret; +} + +static GstPadProbeReturn +playout_item_pad_probe_video_pad_running_time (GstPad * srcpad, + GstPadProbeInfo * info, PlayoutItem * item) +{ + GstEvent *event; + GstBuffer *buffer; + guint64 running_time; + const GstSegment *segment; + + buffer = GST_PAD_PROBE_INFO_BUFFER (info); + event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0); + GST_TRACE ("%s: video sticky event: %" GST_PTR_FORMAT, item->fn, event); + + if (event) { + gst_event_parse_segment (event, &segment); + gst_event_unref (event); + running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME, + GST_BUFFER_PTS (buffer)); + } else { + GST_WARNING ("%s: unable to parse video event for segment; falling back to " + "pts", item->fn); + running_time = GST_BUFFER_PTS (buffer); + } + + if (running_time >= item->running_time) { + /* The video buffer passing through video_mixer now matches the audio buffer + * that passed through audio_mixer when the early switch was requested, so + * this is the time to send an EOS to video_pad, which will complete the + * switch */ + GST_DEBUG ("Sending video EOS to %s", item->fn); + gst_pad_push_event (item->video_pad, gst_event_new_eos ()); + return GST_PAD_PROBE_DROP; + } else { + return GST_PAD_PROBE_PASS; + } +} + +static gboolean +playout_app_activate_next_item_early (PlayoutApp * app) +{ + PlayoutItem *item; + + item = playout_app_get_current_item (app); + if (!item) { + GST_WARNING ("Unable to switch early, no current item"); + return FALSE; + } + + if (item->audio_pad) { + /* If we have an audio pad, EOS audio first, always */ + GST_DEBUG ("Sending audio EOS to %s", item->fn); + gst_pad_push_event (item->audio_pad, gst_event_new_eos ()); + /* We can't send the EOS to the video_pad yet because the running times for + * both mixers are different due to buffering at the audio sink. So we wait + * till the running time of the video_pad matches that of the audio_pad at + * the time the audio EOS was sent, and then EOS video as well. */ + gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER, + (GstPadProbeCallback) playout_item_pad_probe_video_pad_running_time, + item, NULL); + } else if (item->video_pad) { + /* If we have a video pad, EOS audio first, always */ + GST_DEBUG ("Sending video EOS to %s", item->fn); + gst_pad_push_event (item->video_pad, gst_event_new_eos ()); + } else { + g_assert_not_reached (); + } + + /* Return FALSE so this function is called only once */ + return FALSE; +} + +static gboolean +playout_app_play (PlayoutApp * app) +{ + PlayoutItem *item; + + item = app->play_queue->len ? g_ptr_array_index (app->play_queue, 0) : NULL; + if (!item) { + g_printerr ("Nothing to play\n"); + return FALSE; + } + + playout_app_prepare_item (item); + return TRUE; +} + +/* + * playout: An example application to sequentially and seamlessly play a list of + * audio-video or video-only files. + * + * This example application uses the compositor and audiomixer elements combined + * with pad probes to stitch together a list of A/V or V-only files in such + * a way that audio and video glitching is minimised. Mixing A/V and V-only + * files is not supported because it complicates the architecture quite a bit. + * + * Due to the fundamental difference in the representation of audio and video + * data, unless constructed specifically for the purpose of being stitched back, + * the audio and video tracks of files will rarely end at the same PTS. There is + * usually a sync difference of a few frames. This application tries to stitch + * together the audio tracks as perfectly as possible, and duplicates/drops + * video frames if there is an underrun/overrun. Even when audio samples are + * played back-to-back, there might be glitches due to quirks in the decoder. + * + * The list of PlayoutItems can be edited and added to dynamically; except the + * currently-playing item and the next one (which has been prepared already). + */ +int +main (int argc, char **argv) +{ + GstBus *bus; + gint switch_after_ms = 0; + gchar **f, **filenames = NULL; + GOptionEntry options[] = { + {"switch-after", 's', 0, G_OPTION_ARG_INT, &switch_after_ms, "Time after " + "which the next item will be forcibly activated", "MILLISECONDS"}, + {G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &filenames, NULL}, + {NULL} + }; + GOptionContext *ctx; + PlayoutApp *app; + GError *err = NULL; + + ctx = g_option_context_new ("FILENAME1 [FILENAME2] [FILENAME3] ..."); + g_option_context_add_main_entries (ctx, options, NULL); + g_option_context_add_group (ctx, gst_init_get_option_group ()); + + if (!g_option_context_parse (ctx, &argc, &argv, &err)) { + if (err) + g_printerr ("Error initializing: %s\n", err->message); + else + g_printerr ("Error initializing: Unknown error!\n"); + return 1; + } + g_option_context_free (ctx); + + GST_DEBUG_CATEGORY_INIT (playout, "playout", 0, "Playout example app"); + + app = playout_app_new (); + + if (filenames == NULL || *filenames == NULL) { + g_printerr ("Usage: %s FILENAME1 FILENAME2\n", argv[0]); + return 1; + } + + for (f = filenames; f != NULL && *f != NULL; ++f) + playout_app_add_item (app, *f); + + g_strfreev (filenames); + + if (!playout_app_play (app)) + return 1; + + GST_DEBUG ("Setting pipeline to PLAYING"); + + bus = gst_pipeline_get_bus (GST_PIPELINE (app->pipeline)); + gst_bus_add_signal_watch (bus); + g_signal_connect (bus, "message::eos", G_CALLBACK (playout_app_eos), app); + gst_object_unref (bus); + + gst_element_set_state (app->pipeline, GST_STATE_PLAYING); + + if (switch_after_ms) + g_timeout_add (switch_after_ms, + (GSourceFunc) playout_app_activate_next_item_early, app); + + GST_DEBUG ("Running mainloop"); + g_main_loop_run (app->main_loop); + + playout_app_free (app); + + return 0; +} |