/* Copyright (C) 2015 Centricular Ltd * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include #include #define STR_HELPER(x) #x #define STR(x) STR_HELPER(x) /* Change this to set the output resolution */ #define OUTPUT_VIDEO_WIDTH 1280 #define OUTPUT_VIDEO_HEIGHT 720 /* Video and audio caps outputted by the mixers */ #define RAW_AUDIO_CAPS_STR "audio/x-raw, format=(string)S16LE, " \ "layout=(string)interleaved, rate=(int)44100, channels=(int)2, " \ "channel-mask=(bitmask)0x03" #define RAW_VIDEO_CAPS_STR "video/x-raw, width=(int)" STR(OUTPUT_VIDEO_WIDTH) \ ", height=(int)" STR(OUTPUT_VIDEO_HEIGHT) ", framerate=(fraction)25/1, " \ "format=I420, pixel-aspect-ratio=(fraction)1/1, " \ "interlace-mode=(string)progressive" GST_DEBUG_CATEGORY_STATIC (playout); #define GST_CAT_DEFAULT playout typedef enum { PLAYOUT_APP_STATE_READY, /* Newly created */ PLAYOUT_APP_STATE_PLAYING, /* Playing an item */ PLAYOUT_APP_STATE_EOS /* Finished playing, all items EOS */ } PlayoutAppState; typedef struct { /* Application state */ PlayoutAppState state; /* An array of PlayoutItems that will be played in sequence */ GPtrArray *play_queue; /* Index of the currently-playing item */ gint play_queue_current; /* Lock access to the play queue */ GMutex play_queue_lock; GMainLoop *main_loop; /* Pipeline */ GstElement *pipeline; /* Output */ GstElement *video_mixer; GstElement *video_sink; GstVideoRectangle video_orect; /* w/h/x/y of the output */ GstElement *audio_mixer; GstElement *audio_sink; /* The duration of all items that have been played in ns. * Only updates when a new item is activated. */ guint64 elapsed_duration; } PlayoutApp; typedef enum { PLAYOUT_ITEM_STATE_NEW, /* Newly created */ PLAYOUT_ITEM_STATE_PREPARED, /* Prepared and ready to activate */ PLAYOUT_ITEM_STATE_ACTIVATED, /* Activated */ PLAYOUT_ITEM_STATE_FIRST_VBUFFER, /* First video buffer has gone through */ PLAYOUT_ITEM_STATE_AGGREGATING, /* Audio & video buffers are aggregating */ PLAYOUT_ITEM_STATE_EOS /* At least one pad is EOS */ } PlayoutItemState; typedef struct { PlayoutApp *app; PlayoutItemState state; gchar *fn; GstElement *decoder; /* bin with uridecodebin + converters */ /* We just use the first audio stream and ignore the rest (if there is audio) */ GstPad *audio_pad; /* decoder bin audio src ghostpad */ GstPad *video_pad; /* decoder bin video src ghostpad */ GstVideoRectangle video_irect; /* input w/h/x/y of the item */ GstVideoRectangle video_orect; /* output w/h/x/y of the item */ /* When this item has finished preparing and all pads have been connected to * mixers, the pads will be blocked till it's this item's turn to be played */ gulong audio_pad_probe_block_id; gulong video_pad_probe_block_id; /* The current running time of this item; updated with every audio buffer if * this item has audio; otherwise it's updated withe very video buffer */ guint64 running_time; } PlayoutItem; static PlayoutApp *playout_app_new (void); static void playout_app_free (PlayoutApp * app); static PlayoutItem *playout_item_new (PlayoutApp * app, const gchar * fn); static void playout_item_free (PlayoutItem * item); static void playout_app_add_item (PlayoutApp * app, const gchar * fn); static gboolean playout_app_prepare_item (PlayoutItem * item); static gboolean playout_app_activate_item (PlayoutItem * item); static gboolean playout_app_activate_next_item (PlayoutApp * app); static gboolean playout_app_activate_next_item_early (PlayoutApp * app); static PlayoutItem *playout_app_get_current_item (PlayoutApp * app); static gboolean playout_app_remove_item (PlayoutItem * item); static void playout_app_add_audio_sink (PlayoutApp * app) { GstElement *audio_resample, *audio_conv, *queue; /* audiomixer doesn't do conversion yet, so we don't need an output capsfilter * for this branch. The output format is decided by the sink pads, which all * have to have the same format. */ app->audio_mixer = gst_element_factory_make ("audiomixer", "audio_mixer"); audio_conv = gst_element_factory_make ("audioconvert", "mixer_audioconvert"); audio_resample = gst_element_factory_make ("audioresample", "audio_mixer_audioresample"); queue = gst_element_factory_make ("queue", "asink_queue"); app->audio_sink = gst_element_factory_make ("autoaudiosink", NULL); g_object_set (app->audio_sink, "async-handling", TRUE, NULL); gst_bin_add_many (GST_BIN (app->pipeline), app->audio_mixer, audio_conv, audio_resample, queue, app->audio_sink, NULL); gst_element_link_many (app->audio_mixer, audio_conv, audio_resample, queue, app->audio_sink, NULL); if (!gst_element_sync_state_with_parent (app->audio_mixer) || !gst_element_sync_state_with_parent (audio_conv) || !gst_element_sync_state_with_parent (audio_resample) || !gst_element_sync_state_with_parent (queue) || !gst_element_sync_state_with_parent (app->audio_sink)) GST_ERROR ("app: unable to sync audio mixer + sink state with pipeline"); } static PlayoutApp * playout_app_new (void) { GstElement *video_capsfilter, *queue; GstCaps *caps; PlayoutApp *app; app = g_new0 (PlayoutApp, 1); app->state = PLAYOUT_APP_STATE_READY; app->play_queue = g_ptr_array_new_with_free_func ((GDestroyNotify) playout_item_free); app->play_queue_current = -1; g_mutex_init (&app->play_queue_lock); app->main_loop = g_main_loop_new (NULL, FALSE); app->pipeline = gst_pipeline_new ("pipeline"); /* It's best to set a caps filter for the video output format */ app->video_orect.w = OUTPUT_VIDEO_WIDTH; app->video_orect.h = OUTPUT_VIDEO_HEIGHT; app->video_orect.x = 0; app->video_orect.y = 0; app->video_mixer = gst_element_factory_make ("compositor", "video_mixer"); /* Set the background as black; faster while compositing, and allows us to * rescale videos with a different aspect ratio than the output in a way that * adds black borders automatically */ g_object_set (app->video_mixer, "background", 1, NULL); queue = gst_element_factory_make ("queue", "vsink_queue"); app->video_sink = gst_element_factory_make ("autovideosink", NULL); g_object_set (app->video_sink, "async-handling", TRUE, NULL); video_capsfilter = gst_element_factory_make ("capsfilter", "video_mixer_capsfilter"); caps = gst_caps_from_string (RAW_VIDEO_CAPS_STR); g_object_set (video_capsfilter, "caps", caps, NULL); gst_caps_unref (caps); gst_bin_add_many (GST_BIN (app->pipeline), app->video_mixer, video_capsfilter, queue, app->video_sink, NULL); gst_element_link_many (app->video_mixer, video_capsfilter, queue, app->video_sink, NULL); return app; } static void playout_app_free (PlayoutApp * app) { GST_DEBUG ("Freeing app"); g_ptr_array_unref (app->play_queue); g_main_loop_unref (app->main_loop); gst_element_set_state (app->pipeline, GST_STATE_NULL); gst_object_unref (app->pipeline); g_free (app); } static void playout_app_eos (GstBus * bus, GstMessage * msg, PlayoutApp * app) { g_print ("All streams EOS, exiting...\n"); g_main_loop_quit (app->main_loop); } static PlayoutItem * playout_item_new (PlayoutApp * app, const gchar * fn) { PlayoutItem *item = g_new0 (PlayoutItem, 1); item->app = app; item->state = PLAYOUT_ITEM_STATE_NEW; item->fn = g_strdup (fn); return item; } /* Unlink and release the pad */ static gboolean playout_remove_pad (GstPad * srcpad) { GstPad *sinkpad; GstElement *mixer; sinkpad = gst_pad_get_peer (srcpad); if (!sinkpad) return FALSE; if (!gst_pad_unlink (srcpad, sinkpad)) return FALSE; mixer = gst_pad_get_parent_element (sinkpad); gst_element_release_request_pad (mixer, sinkpad); GST_DEBUG ("Released some pad"); gst_object_unref (sinkpad); gst_object_unref (mixer); return FALSE; } static GstPadProbeReturn playout_item_pad_probe_blocked (GstPad * srcpad, GstPadProbeInfo * info, PlayoutItem * item) { if (srcpad == item->audio_pad) { item->audio_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info); } else if (srcpad == item->video_pad) { item->video_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info); } else { g_assert_not_reached (); } return GST_PAD_PROBE_OK; } static GstPadProbeReturn playout_item_pad_probe_pad_running_time (GstPad * srcpad, GstPadProbeInfo * info, PlayoutItem * item) { GstEvent *event; GstBuffer *buffer; guint64 running_time; const GstSegment *segment; buffer = GST_PAD_PROBE_INFO_BUFFER (info); event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0); GST_TRACE ("%s: pad sticky event: %" GST_PTR_FORMAT, item->fn, event); if (event) { gst_event_parse_segment (event, &segment); gst_event_unref (event); running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME, GST_BUFFER_PTS (buffer)); } else { GST_WARNING ("%s: unable to parse event for segment; falling back to pts. " "Output will probably have glitches.", item->fn); running_time = GST_BUFFER_PTS (buffer); } item->running_time = running_time + GST_BUFFER_DURATION (buffer); GST_TRACE ("%s: running time is %" G_GUINT64_FORMAT ", duration is %" G_GUINT64_FORMAT, item->fn, item->running_time, GST_BUFFER_DURATION (buffer)); return GST_PAD_PROBE_PASS; } static GstPadProbeReturn playout_item_pad_probe_video_pad_eos_on_buffer (GstPad * srcpad, GstPadProbeInfo * info, PlayoutItem * prev_item) { PlayoutItem *current_item; current_item = playout_app_get_current_item (prev_item->app); if (!current_item) return GST_PAD_PROBE_REMOVE; /* Step through the item's states as buffers pass through. The first buffer * will be taken by the video_mixer, and kept till the audio running time * matches the video buffer running time. When the second buffer gets through, * we know that this pad has begun aggregating. */ switch (current_item->state) { case PLAYOUT_ITEM_STATE_NEW: case PLAYOUT_ITEM_STATE_PREPARED: GST_DEBUG ("%s: new/prepared", current_item->fn); break; case PLAYOUT_ITEM_STATE_ACTIVATED: GST_DEBUG ("%s: activated -> first vbuffer", current_item->fn); current_item->state = PLAYOUT_ITEM_STATE_FIRST_VBUFFER; break; case PLAYOUT_ITEM_STATE_FIRST_VBUFFER: GST_DEBUG ("%s: first vbuffer -> aggregating", current_item->fn); current_item->state = PLAYOUT_ITEM_STATE_AGGREGATING; gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info)); /* Item is aggregating, release the previous item's video pad */ goto release; break; case PLAYOUT_ITEM_STATE_EOS: return GST_PAD_PROBE_REMOVE; default: g_assert_not_reached (); } return GST_PAD_PROBE_PASS; release: { playout_remove_pad (prev_item->video_pad); GST_DEBUG ("%s: released video pad", prev_item->fn); prev_item->video_pad = NULL; /* If there's no audio pad, or if the audio pad is already EOS, we can * remove this item from the queue which will free it. Need to free the * item from the main thread because it causes the item's decoder bin * to be removed from the pipeline, which cannot be done in the * streaming thread */ if (prev_item->audio_pad == NULL) { GST_DEBUG ("%s: queued item removal (last pad is video)", prev_item->fn); g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item, prev_item); } /* Pad probe has already been removed above */ return GST_PAD_PROBE_PASS; } } /* This is called on EOS for both item->audio_pad and item->video_pad * * FIXME: Add locking. Both pads could go EOS at the exact same time. */ static GstPadProbeReturn playout_item_pad_probe_event (GstPad * srcpad, GstPadProbeInfo * info, PlayoutItem * item) { GstEventType type; gboolean ret = TRUE; GstPadProbeReturn probe_ret = GST_PAD_PROBE_DROP; type = GST_EVENT_TYPE (GST_PAD_PROBE_INFO_DATA (info)); if (type != GST_EVENT_EOS) return GST_PAD_PROBE_PASS; /* We might get two EOSes on this pad if we send an artificial EOS. Remove * the probe so this is only called once for each pad */ gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info)); GST_DEBUG ("%s: recvd some EOS", item->fn); if (item->state != PLAYOUT_ITEM_STATE_EOS) { /* We have more than one pad per item (video + audio item), and this is the * first pad to go EOS or we have only one pad per item, and that pad has * gone EOS. For the first case, the other pad might still have some buffers * to output before going EOS, but we need to activate the next item and * start outputting buffers from that immediately. */ /* Update the total elapsed duration from the item's current running time */ item->app->elapsed_duration += item->running_time; GST_DEBUG ("%s: activating next item", item->fn); /* Activate the next item if and only if this is the first pad to go EOS */ ret = playout_app_activate_next_item (item->app); if (!ret) { GST_DEBUG ("%s: App is going EOS", item->fn); item->state = PLAYOUT_ITEM_STATE_EOS; item->app->state = PLAYOUT_APP_STATE_EOS; /* If we couldn't activate the next item, pass the EOS event onward, * ending the stream */ probe_ret = GST_PAD_PROBE_PASS; } } g_assert (srcpad != NULL); if (srcpad == item->audio_pad) { GST_DEBUG ("%s: audio pad was EOS", item->fn); if (item->app->state != PLAYOUT_APP_STATE_EOS) { /* While activating the next item, we ensure that there's no offset mismatch * which would cause audiomixer to output silence, so we can release the * previous item's audio request pad here. We also unlink the audio pad; * nothing else is needed from it */ playout_remove_pad (srcpad); GST_DEBUG ("%s: released audio pad", item->fn); /* If there's no video pad, or if the video pad is already EOS, we can * remove this item from the queue which will free it. Need to free the * item from the main thread because it causes the item's decoder bin * to be removed from the pipeline, which cannot be done in the * streaming thread */ if (item->video_pad == NULL) { GST_DEBUG ("%s: queued item removal (last pad is audio)", item->fn); g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item, item); } } else { /* If this is the last pad on audio_mixer, let the EOS go through * before unlinking/releasing the pad. This should happen within 500ms. */ g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad); GST_DEBUG ("%s: queued audio pad release", item->fn); if (item->video_pad == NULL) { /* Unlike above, we need to wait till the pad is removed before removing * the item from the app, so we queue it for 100ms afterwards */ GST_DEBUG ("%s: queued last item removal (last pad is audio)", item->fn); g_timeout_add (600, (GSourceFunc) playout_app_remove_item, item); } } item->audio_pad = NULL; } else if (srcpad == item->video_pad) { GST_DEBUG ("%s: video pad was EOS", item->fn); if (item->audio_pad != NULL) GST_WARNING ("%s: video pad went EOS before audio pad! " "There will be audio/video glitches while switching.", item->fn); if (item->app->state != PLAYOUT_APP_STATE_EOS) { PlayoutItem *next_item; next_item = playout_app_get_current_item (item->app); GST_DEBUG ("%s: next item is %s, %i/%i", item->fn, next_item->fn, next_item->state, PLAYOUT_ITEM_STATE_ACTIVATED); g_assert (next_item != NULL); /* If there's another item being activated, release this video pad only * when the next item's video pad starts being aggregated; that happens * when this probe receives its 2nd buffer from the next item */ gst_pad_add_probe (next_item->video_pad, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback) playout_item_pad_probe_video_pad_eos_on_buffer, item, NULL); } else { /* If this is the last pad on video_mixer, let the EOS go through * before unlinking/releasing the pad. This should happen within 500ms. */ g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad); GST_DEBUG ("%s: queued video pad release", item->fn); item->video_pad = NULL; } probe_ret = GST_PAD_PROBE_PASS; } else { g_assert_not_reached (); } item->state = PLAYOUT_ITEM_STATE_EOS; /* NOTE: If the srcpad has been unlinked, the return value is useless */ return probe_ret; } /* On the "pad-added" signal of uridecodebin, add converters and connect to * audio/video mixers */ static void playout_item_new_pad (GstElement * uridecodebin, GstPad * pad, PlayoutItem * item) { GstStructure *s; GstCaps *caps; GstPad *sinkpad, *srcpad; GstElement *queue; GstPadProbeType block_probe_type; caps = gst_pad_get_current_caps (pad); s = gst_caps_get_structure (caps, 0); GST_DEBUG ("%s: new pad: %p, caps: %s", item->fn, pad, gst_structure_get_name (s)); if (gst_structure_has_name (s, "audio/x-raw")) { if (item->audio_pad != NULL) /* Ignore all audio pads after the first one */ goto out; goto audio; } else if (gst_structure_has_name (s, "video/x-raw")) { if (item->video_pad != NULL) /* Ignore all video pads after the first one */ goto out; goto video; } else { goto out; } audio: { GstCaps *wanted_caps; GstElement *audioconvert, *audioresample, *capsfilter; /* Audio pad found; add audio mixer and audio sink to the pipeline. * NOTE: If any items after this do not have an audio pad, the pipeline will * mess up because the audio sink will not receive any data. */ if (item->app->audio_sink == NULL) playout_app_add_audio_sink (item->app); wanted_caps = gst_caps_from_string (RAW_AUDIO_CAPS_STR); if (!gst_caps_is_equal (caps, wanted_caps)) { GST_DEBUG ("%s: converting audio caps", item->fn); /* We need to convert the audio to the wanted format because * audiomixer doesn't do format conversion */ audioresample = gst_element_factory_make ("audioresample", NULL); audioconvert = gst_element_factory_make ("audioconvert", NULL); capsfilter = gst_element_factory_make ("capsfilter", NULL); g_object_set (capsfilter, "caps", wanted_caps, NULL); queue = gst_element_factory_make ("queue", NULL); gst_bin_add_many (GST_BIN (item->decoder), audioresample, audioconvert, capsfilter, queue, NULL); sinkpad = gst_element_get_static_pad (audioresample, "sink"); gst_pad_link (pad, sinkpad); gst_object_unref (sinkpad); gst_element_link_many (audioresample, audioconvert, capsfilter, queue, NULL); srcpad = gst_element_get_static_pad (queue, "src"); if (!gst_element_sync_state_with_parent (audioresample) || !gst_element_sync_state_with_parent (audioconvert) || !gst_element_sync_state_with_parent (capsfilter) || !gst_element_sync_state_with_parent (queue)) { GST_ERROR ("%s: unable to sync audio converter state with decoder", item->fn); goto out; } } else { queue = gst_element_factory_make ("queue", NULL); gst_bin_add (GST_BIN (item->decoder), queue); sinkpad = gst_element_get_static_pad (queue, "sink"); gst_pad_link (pad, sinkpad); gst_object_unref (sinkpad); srcpad = gst_element_get_static_pad (queue, "src"); if (!gst_element_sync_state_with_parent (queue)) { GST_ERROR ("%s: unable to sync audio queue state with decoder", item->fn); goto out; } } gst_caps_unref (wanted_caps); /* Convert the audioconvert src pad to a ghostpad on the bin */ item->audio_pad = gst_ghost_pad_new (NULL, srcpad); gst_pad_set_active (item->audio_pad, TRUE); gst_element_add_pad (item->decoder, item->audio_pad); gst_object_unref (srcpad); srcpad = item->audio_pad; GST_DEBUG ("%s: created audio pad", item->fn); goto done; } video: { if (!gst_structure_get_int (s, "width", &item->video_irect.w) || !gst_structure_get_int (s, "height", &item->video_irect.h)) GST_WARNING ("%s: unable to set width/height from caps", item->fn); item->video_irect.x = item->video_irect.y = 0; queue = gst_element_factory_make ("queue", "video-decoder-queue-%u"); gst_bin_add (GST_BIN (item->decoder), queue); if (!gst_element_sync_state_with_parent (queue)) { GST_ERROR ("%s: unable to sync video queue state with decoder", item->fn); goto out; } sinkpad = gst_element_get_static_pad (queue, "sink"); gst_pad_link (pad, sinkpad); gst_object_unref (sinkpad); /* Convert the queue src pad to a ghostpad on the bin */ srcpad = gst_element_get_static_pad (queue, "src"); item->video_pad = gst_ghost_pad_new (NULL, srcpad); gst_pad_set_active (item->video_pad, TRUE); gst_element_add_pad (item->decoder, item->video_pad); gst_object_unref (srcpad); srcpad = item->video_pad; GST_DEBUG ("%s: created video pad", item->fn); goto done; } done: /* We let events and queries through */ block_probe_type = GST_PAD_PROBE_TYPE_BLOCK | GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST; /* If the app is already playing an item, block everything except queries * till we need to play this item */ if (item->app->state != PLAYOUT_APP_STATE_READY) gst_pad_add_probe (srcpad, block_probe_type, (GstPadProbeCallback) playout_item_pad_probe_blocked, item, NULL); /* Probe events for EOS */ gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, (GstPadProbeCallback) playout_item_pad_probe_event, item, NULL); out: gst_caps_unref (caps); } /* All pads on uridecodebin have finished being populated; the item has been * prepared and is ready to be activated */ static void playout_item_no_more_pads (GstElement * uridecodebin, PlayoutItem * item) { /* Set a buffer pad probe that constantly updates the item's * elapsed_duration using the duration of each audio buffer */ if (item->audio_pad) { gst_pad_add_probe (item->audio_pad, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback) playout_item_pad_probe_pad_running_time, item, NULL); } else if (item->video_pad) { gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback) playout_item_pad_probe_pad_running_time, item, NULL); } else { GST_ERROR ("%s: no pads were generated! Can't continue playing!", item->fn); return; } item->state = PLAYOUT_ITEM_STATE_PREPARED; GST_DEBUG ("%s: prepared", item->fn); if (item->app->state != PLAYOUT_APP_STATE_READY) /* This item will be activated when the previous one is EOS */ return; GST_DEBUG ("Application isn't already playing; activate the item and prepare" " the next one"); playout_app_activate_item (item); item->state = PLAYOUT_ITEM_STATE_ACTIVATED; item->app->state = PLAYOUT_APP_STATE_PLAYING; if (item->app->play_queue->len > 1) playout_app_prepare_item (g_ptr_array_index (item->app->play_queue, 1)); } static GstElement * playout_item_create_decoder (PlayoutItem * item) { GstElement *bin, *dec; GError *err = NULL; gchar *uri; uri = gst_filename_to_uri (item->fn, &err); if (err != NULL) { GST_WARNING ("Could not convert '%s' to uri: %s", item->fn, err->message); g_clear_error (&err); return NULL; } bin = gst_bin_new (NULL); dec = gst_element_factory_make ("uridecodebin", NULL); g_object_set (dec, "uri", uri, NULL); g_free (uri); gst_bin_add (GST_BIN (bin), dec); g_signal_connect (dec, "pad-added", G_CALLBACK (playout_item_new_pad), item); g_signal_connect (dec, "no-more-pads", G_CALLBACK (playout_item_no_more_pads), item); return bin; } static void playout_item_free (PlayoutItem * item) { GST_DEBUG ("Entering free"); switch (gst_element_set_state (item->decoder, GST_STATE_NULL)) { case GST_STATE_CHANGE_FAILURE: GST_ERROR ("%s: Unable to change state to NULL", item->fn); break; case GST_STATE_CHANGE_SUCCESS: GST_DEBUG ("%s: State change success", item->fn); break; default: GST_DEBUG ("%s: Some async/no-preroll", item->fn); } gst_bin_remove (GST_BIN (item->app->pipeline), item->decoder); GST_DEBUG ("%s: bin removed", item->fn); g_free (item->fn); g_free (item); GST_DEBUG ("item freed"); } static guint64 playout_item_pad_get_segment_time (GstPad * srcpad) { GstEvent *event; const GstSegment *segment; event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0); if (!event) return 0; gst_event_parse_segment (event, &segment); gst_event_unref (event); return segment->time; } static void playout_app_add_item (PlayoutApp * app, const gchar * fn) { PlayoutItem *item; item = playout_item_new (app, fn); g_mutex_lock (&app->play_queue_lock); g_ptr_array_add (app->play_queue, item); g_mutex_unlock (&app->play_queue_lock); } static gboolean playout_app_remove_item (PlayoutItem * item) { PlayoutApp *app; GST_DEBUG ("%s: removing and freeing", item->fn); app = item->app; g_mutex_lock (&app->play_queue_lock); g_ptr_array_remove (app->play_queue, item); if (item->state >= PLAYOUT_ITEM_STATE_ACTIVATED) /* Removed item was playing; decrement the current-play-queue index */ app->play_queue_current--; g_mutex_unlock (&app->play_queue_lock); /* Don't call this again */ return FALSE; } static PlayoutItem * playout_app_get_current_item (PlayoutApp * app) { if (app->play_queue_current < 0 || app->play_queue->len < (app->play_queue_current + 1)) return NULL; return g_ptr_array_index (app->play_queue, app->play_queue_current); } static gboolean playout_app_prepare_item (PlayoutItem * item) { PlayoutApp *app = item->app; if (item->decoder != NULL) return TRUE; item->decoder = playout_item_create_decoder (item); if (item->decoder == NULL) return FALSE; gst_bin_add (GST_BIN (app->pipeline), item->decoder); if (!gst_element_sync_state_with_parent (item->decoder)) { GST_ERROR ("%s: unable to sync state with parent", item->fn); return FALSE; } GST_DEBUG ("%s: preparing", item->fn); /* All further processing is done in the "no-more-pads" callback of * uridecodebin */ return TRUE; } /* Called exactly once for each item */ static gboolean playout_app_activate_item (PlayoutItem * item) { GstPad *sinkpad; guint64 segment_time; PlayoutApp *app = item->app; if (item->state != PLAYOUT_ITEM_STATE_PREPARED) { GST_ERROR ("Item %s is not ready to be activated!", item->fn); return FALSE; } if (!item->audio_pad && !item->video_pad) { GST_ERROR ("Item %s has no pads! Can't activate it!", item->fn); return FALSE; } /* Hook up to mixers and remove the probes blocking the pads */ if (item->audio_pad) { GST_DEBUG ("%s: hooking up audio pad to mixer", item->fn); sinkpad = gst_element_get_request_pad (app->audio_mixer, "sink_%u"); gst_pad_link (item->audio_pad, sinkpad); segment_time = playout_item_pad_get_segment_time (item->audio_pad); if (segment_time > 0) { /* If the segment time is > 0, the new pad wants audiomixer to output audio * silence for that duration. This will cause audio glitches, so we move * the pad offset back by that amount and tell audiomixer to start mixing * our buffers immediately. */ GST_DEBUG ("%s: subtracting segment time %" G_GUINT64_FORMAT " from " "elapsed duration before setting it as the pad offset", item->fn, segment_time); if (app->elapsed_duration > segment_time) app->elapsed_duration -= segment_time; else app->elapsed_duration = 0; } if (app->elapsed_duration > 0) { GST_DEBUG ("%s: set audio pad offset to %" G_GUINT64_FORMAT "ms", item->fn, app->elapsed_duration / GST_MSECOND); gst_pad_set_offset (item->audio_pad, app->elapsed_duration); } if (item->audio_pad_probe_block_id > 0) { GST_DEBUG ("%s: removing audio pad block probe", item->fn); gst_pad_remove_probe (item->audio_pad, item->audio_pad_probe_block_id); } gst_object_unref (sinkpad); } if (item->video_pad) { GST_DEBUG ("%s: hooking up video pad to mixer", item->fn); sinkpad = gst_element_get_request_pad (app->video_mixer, "sink_%u"); /* Get new height/width/xpos/ypos such that the video scales up or down to * fit within the output video size without any cropping */ gst_video_sink_center_rect (item->video_irect, item->app->video_orect, &item->video_orect, TRUE); GST_DEBUG ("%s: w: %i, h: %i, x: %i, y: %i\n", item->fn, item->video_orect.w, item->video_orect.h, item->video_orect.x, item->video_orect.y); g_object_set (sinkpad, "width", item->video_orect.w, "height", item->video_orect.h, "xpos", item->video_orect.x, "ypos", item->video_orect.y, NULL); /* If this is not the last item, on EOS, continue to aggregate using the * last buffer till the pad is released */ if (item->app->play_queue->len != (item->app->play_queue_current + 2)) g_object_set (sinkpad, "ignore-eos", TRUE, NULL); else GST_DEBUG ("%s: last item, not setting ignore-eos", item->fn); gst_pad_link (item->video_pad, sinkpad); if (app->elapsed_duration > 0) { GST_DEBUG ("%s: set video pad offset to %" G_GUINT64_FORMAT "ms", item->fn, app->elapsed_duration / GST_MSECOND); gst_pad_set_offset (item->video_pad, app->elapsed_duration); } if (item->video_pad_probe_block_id > 0) { GST_DEBUG ("%s: removing video pad block probe", item->fn); gst_pad_remove_probe (item->video_pad, item->video_pad_probe_block_id); } gst_object_unref (sinkpad); } item->state = PLAYOUT_ITEM_STATE_ACTIVATED; g_mutex_lock (&item->app->play_queue_lock); item->app->play_queue_current++; g_mutex_unlock (&item->app->play_queue_lock); GST_DEBUG ("%s: activated", item->fn); return TRUE; } /* Activate the next item, and prepare the one after that for later activation */ static gboolean playout_app_activate_next_item (PlayoutApp * app) { PlayoutItem *item; gboolean ret; if (app->play_queue->len < (app->play_queue_current + 2)) { g_print ("No more items to play\n"); return FALSE; } item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1); ret = playout_app_activate_item (item); if (!ret) { /* Tell caller, who can then decide whether to skip or error out */ GST_ERROR ("%s: unable to activate", item->fn); return FALSE; } if (app->play_queue->len > (app->play_queue_current + 1)) { item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1); /* FIXME: What if this fails? Prepare the next one in the queue? */ ret = playout_app_prepare_item (item); if (!ret) GST_ERROR ("%s: unable to prepare", item->fn); } return ret; } static GstPadProbeReturn playout_item_pad_probe_video_pad_running_time (GstPad * srcpad, GstPadProbeInfo * info, PlayoutItem * item) { GstEvent *event; GstBuffer *buffer; guint64 running_time; const GstSegment *segment; buffer = GST_PAD_PROBE_INFO_BUFFER (info); event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0); GST_TRACE ("%s: video sticky event: %" GST_PTR_FORMAT, item->fn, event); if (event) { gst_event_parse_segment (event, &segment); gst_event_unref (event); running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME, GST_BUFFER_PTS (buffer)); } else { GST_WARNING ("%s: unable to parse video event for segment; falling back to " "pts", item->fn); running_time = GST_BUFFER_PTS (buffer); } if (running_time >= item->running_time) { /* The video buffer passing through video_mixer now matches the audio buffer * that passed through audio_mixer when the early switch was requested, so * this is the time to send an EOS to video_pad, which will complete the * switch */ GST_DEBUG ("Sending video EOS to %s", item->fn); gst_pad_push_event (item->video_pad, gst_event_new_eos ()); return GST_PAD_PROBE_DROP; } else { return GST_PAD_PROBE_PASS; } } static gboolean playout_app_activate_next_item_early (PlayoutApp * app) { PlayoutItem *item; item = playout_app_get_current_item (app); if (!item) { GST_WARNING ("Unable to switch early, no current item"); return FALSE; } if (item->audio_pad) { /* If we have an audio pad, EOS audio first, always */ GST_DEBUG ("Sending audio EOS to %s", item->fn); gst_pad_push_event (item->audio_pad, gst_event_new_eos ()); /* We can't send the EOS to the video_pad yet because the running times for * both mixers are different due to buffering at the audio sink. So we wait * till the running time of the video_pad matches that of the audio_pad at * the time the audio EOS was sent, and then EOS video as well. */ gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback) playout_item_pad_probe_video_pad_running_time, item, NULL); } else if (item->video_pad) { /* If we have a video pad, EOS audio first, always */ GST_DEBUG ("Sending video EOS to %s", item->fn); gst_pad_push_event (item->video_pad, gst_event_new_eos ()); } else { g_assert_not_reached (); } /* Return FALSE so this function is called only once */ return FALSE; } static gboolean playout_app_play (PlayoutApp * app) { PlayoutItem *item; item = app->play_queue->len ? g_ptr_array_index (app->play_queue, 0) : NULL; if (!item) { g_printerr ("Nothing to play\n"); return FALSE; } playout_app_prepare_item (item); return TRUE; } /* * playout: An example application to sequentially and seamlessly play a list of * audio-video or video-only files. * * This example application uses the compositor and audiomixer elements combined * with pad probes to stitch together a list of A/V or V-only files in such * a way that audio and video glitching is minimised. Mixing A/V and V-only * files is not supported because it complicates the architecture quite a bit. * * Due to the fundamental difference in the representation of audio and video * data, unless constructed specifically for the purpose of being stitched back, * the audio and video tracks of files will rarely end at the same PTS. There is * usually a sync difference of a few frames. This application tries to stitch * together the audio tracks as perfectly as possible, and duplicates/drops * video frames if there is an underrun/overrun. Even when audio samples are * played back-to-back, there might be glitches due to quirks in the decoder. * * The list of PlayoutItems can be edited and added to dynamically; except the * currently-playing item and the next one (which has been prepared already). */ int main (int argc, char **argv) { GstBus *bus; gint switch_after_ms = 0; gchar **f, **filenames = NULL; GOptionEntry options[] = { {"switch-after", 's', 0, G_OPTION_ARG_INT, &switch_after_ms, "Time after " "which the next file will be forcibly activated", "MILLISECONDS"}, {G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &filenames, NULL, "FILENAME1 [FILENAME2] [FILENAME3] ..."}, {NULL} }; GOptionContext *ctx; PlayoutApp *app; GError *err = NULL; ctx = g_option_context_new (NULL); g_option_context_set_summary (ctx, "An example application to sequentially " "and seamlessly play a list of audio-video or video-only files."); g_option_context_add_main_entries (ctx, options, NULL); g_option_context_add_group (ctx, gst_init_get_option_group ()); if (!g_option_context_parse (ctx, &argc, &argv, &err)) { if (err) g_printerr ("Error initializing: %s\n", err->message); else g_printerr ("Error initializing: Unknown error!\n"); g_option_context_free (ctx); g_clear_error (&err); return 1; } if (filenames == NULL || *filenames == NULL) { g_printerr ("%s", g_option_context_get_help (ctx, TRUE, NULL)); return 1; } g_option_context_free (ctx); GST_DEBUG_CATEGORY_INIT (playout, "playout", 0, "Playout example app"); app = playout_app_new (); for (f = filenames; f != NULL && *f != NULL; ++f) playout_app_add_item (app, *f); g_strfreev (filenames); if (!playout_app_play (app)) return 1; GST_DEBUG ("Setting pipeline to PLAYING"); bus = gst_pipeline_get_bus (GST_PIPELINE (app->pipeline)); gst_bus_add_signal_watch (bus); g_signal_connect (bus, "message::eos", G_CALLBACK (playout_app_eos), app); gst_object_unref (bus); gst_element_set_state (app->pipeline, GST_STATE_PLAYING); if (switch_after_ms) g_timeout_add (switch_after_ms, (GSourceFunc) playout_app_activate_next_item_early, app); GST_DEBUG ("Running mainloop"); g_main_loop_run (app->main_loop); playout_app_free (app); return 0; }