path: root/gst/fsrtpconference/fs-rtp-bitrate-adapter.c

/*
 * Farstream Voice+Video library
 *
 *  Copyright 2011 Collabora Ltd,
 *  Copyright 2011 Nokia Corporation
 *   @author: Olivier Crete <olivier.crete@collabora.co.uk>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 */


#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include "fs-rtp-bitrate-adapter.h"

#include <math.h>

/* This is a magical value that smarter people discovered */
/* This is H.264... other codecs (H.265 / VP9 ) will have different numbers */
#define  H264_MAX_PIXELS_PER_BIT 25

GST_DEBUG_CATEGORY_STATIC (fs_rtp_bitrate_adapter_debug);
#define GST_CAT_DEFAULT fs_rtp_bitrate_adapter_debug

static GstStaticPadTemplate fs_rtp_bitrate_adapter_sink_template =
    GST_STATIC_PAD_TEMPLATE ("sink",
        GST_PAD_SINK,
        GST_PAD_ALWAYS,
        GST_STATIC_CAPS_ANY);

static GstStaticPadTemplate fs_rtp_bitrate_adapter_src_template =
    GST_STATIC_PAD_TEMPLATE ("src",
        GST_PAD_SRC,
        GST_PAD_ALWAYS,
        GST_STATIC_CAPS_ANY);
enum
{
  PROP_0,
  PROP_BITRATE,
  PROP_INTERVAL,
};

#define PROP_INTERVAL_DEFAULT (10 * GST_SECOND)
#define PROP_BITRATE_DEFAULT (G_MAXUINT)

static void fs_rtp_bitrate_adapter_finalize (GObject *object);
static void fs_rtp_bitrate_adapter_set_property (GObject *object,
    guint prop_id,
    const GValue *value,
    GParamSpec *pspec);


G_DEFINE_TYPE (FsRtpBitrateAdapter, fs_rtp_bitrate_adapter, GST_TYPE_ELEMENT);

static GstFlowReturn fs_rtp_bitrate_adapter_chain (GstPad *pad,
    GstObject *parent, GstBuffer *buffer);
static gboolean fs_rtp_bitrate_adapter_query (GstPad *pad, GstObject *parent,
    GstQuery *query);

static GstStateChangeReturn
fs_rtp_bitrate_adapter_change_state (GstElement *element,
    GstStateChange transition);

static void
fs_rtp_bitrate_adapter_class_init (FsRtpBitrateAdapterClass *klass)
{
  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
  GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass);

  gobject_class->set_property = fs_rtp_bitrate_adapter_set_property;
  gobject_class->finalize = fs_rtp_bitrate_adapter_finalize;

  gstelement_class->change_state = fs_rtp_bitrate_adapter_change_state;

  GST_DEBUG_CATEGORY_INIT
      (fs_rtp_bitrate_adapter_debug, "fsrtpbitrateadapter", 0,
          "fsrtpbitrateadapter element");

  gst_element_class_set_details_simple (gstelement_class,
      "Farstream RTP Video Bitrate adater",
      "Generic",
      "Filter that can modify the resolution and framerate based"
      " on the bitrate",
      "Olivier Crete <olivier.crete@collabora.co.uk>");

  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&fs_rtp_bitrate_adapter_sink_template));
  gst_element_class_add_pad_template (gstelement_class,
      gst_static_pad_template_get (&fs_rtp_bitrate_adapter_src_template));

  g_object_class_install_property (gobject_class,
      PROP_BITRATE,
      g_param_spec_uint ("bitrate",
          "Bitrate to adapt for",
          "The bitrate to adapt for (MAXUINT means no adaption)",
          0, G_MAXUINT, PROP_BITRATE_DEFAULT,
          G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));

 g_object_class_install_property (gobject_class,
      PROP_INTERVAL,
      g_param_spec_uint64 ("interval",
          "Minimum interval before adaptation",
          "The minimum interval before adapting after a change",
          0, G_MAXUINT64, PROP_INTERVAL_DEFAULT,
          G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS));
}

struct BitratePoint
{
  GstClockTime timestamp;
  guint bitrate;
};

static struct BitratePoint *
bitrate_point_new (GstClockTime timestamp, guint bitrate)
{
  struct BitratePoint *bp = g_slice_new (struct BitratePoint);

  bp->timestamp = timestamp;
  bp->bitrate = bitrate;

  return bp;
}

static void
bitrate_point_free (struct BitratePoint *bp)
{
  g_slice_free (struct BitratePoint, bp);
}


static void
fs_rtp_bitrate_adapter_init (FsRtpBitrateAdapter *self)
{
  self->sinkpad = gst_pad_new_from_static_template (
    &fs_rtp_bitrate_adapter_sink_template, "sink");
  gst_pad_set_chain_function (self->sinkpad, fs_rtp_bitrate_adapter_chain);
  gst_pad_set_query_function (self->sinkpad, fs_rtp_bitrate_adapter_query);
  GST_PAD_SET_PROXY_CAPS (self->sinkpad);
  gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);

  self->srcpad = gst_pad_new_from_static_template (
    &fs_rtp_bitrate_adapter_src_template, "src");
  gst_pad_set_query_function (self->sinkpad, fs_rtp_bitrate_adapter_query);
  gst_element_add_pad (GST_ELEMENT (self), self->srcpad);

  g_queue_init (&self->bitrate_history);
  self->system_clock = gst_system_clock_obtain ();
  self->interval = PROP_INTERVAL_DEFAULT;

  self->bitrate = PROP_BITRATE_DEFAULT;
  self->last_bitrate = G_MAXUINT;
}

static void
fs_rtp_bitrate_adapter_finalize (GObject *object)
{
  FsRtpBitrateAdapter *self = FS_RTP_BITRATE_ADAPTER (object);

  if (self->system_clock)
    gst_object_unref (self->system_clock);

  g_queue_foreach (&self->bitrate_history, (GFunc) bitrate_point_free, NULL);
  g_queue_clear(&self->bitrate_history);

  G_OBJECT_CLASS (fs_rtp_bitrate_adapter_parent_class)->finalize (object);
}

struct Resolution {
  guint width;
  guint height;
};

static const struct Resolution one_on_one_resolutions[] =
{
  {1920, 1200},
  {1920, 1080},
  {1600, 1200},
  {1680, 1050},
  {1280, 800},
  {1280, 768},
  {1280, 720},
  {1024, 768},
  {800, 600},
  {854, 480},
  {800, 480},
  {640, 480},
  {320, 240},
  {160, 120},
  {128, 96},
  {1, 1}
};

static const struct Resolution twelve_on_eleven_resolutions[] =
{
  {1480, 1152},
  {704, 576},
  {352, 288},
  {176, 144},
  {1, 1}
};

static void
video_caps_add (GstCaps *caps, const gchar *media_type,
    guint min_framerate, guint max_framerate, guint width, guint height,
    guint par_n, guint par_d)
{
  GstStructure *s;

  s = gst_structure_new (media_type,
      "pixel-aspect-ratio", GST_TYPE_FRACTION, par_n, par_d,
      "width", G_TYPE_INT, width,
      "height", G_TYPE_INT, height,
      NULL);

  gst_structure_set (s,
      "framerate", GST_TYPE_FRACTION_RANGE, min_framerate, 1,
      max_framerate, 1, NULL);

  gst_caps_append_structure (caps, s);
}

static void
add_one_resolution_inner (GstCaps *caps, const gchar *media_type,
    guint min_framerate, guint max_framerate, guint width, guint height,
    guint par_n, guint par_d)
{
  video_caps_add (caps, media_type, min_framerate, max_framerate,
      width, height, par_n, par_d);
}

static void
add_one_resolution (const gchar *media_type, GstCaps *caps,
    GstCaps *lower_caps,
    GstCaps *extra_low_caps,
    guint max_pixels_per_second,
    guint width, guint height,
    guint par_n, guint par_d)
{
  guint pixels_per_frame = width * height;
  guint max_framerate = max_pixels_per_second / pixels_per_frame;

  /* 66 as the max framerate is a arbitrary number that I'm getting from
   * being 2/3 of 666 which is clearly evil
   */

  if (max_framerate >= 20)
  {
    add_one_resolution_inner (caps, media_type, 20, 66, width, height,
        par_n, par_d);
    add_one_resolution_inner (lower_caps, media_type, 10, 66, width, height,
        par_n, par_d);
    add_one_resolution_inner (extra_low_caps, media_type, 1, 66, width, height,
        par_n, par_d);
  }
  else if (max_framerate >= 10)
  {
    add_one_resolution_inner (lower_caps, media_type, 10, 66, width, height,
        par_n, par_d);
    add_one_resolution_inner (extra_low_caps, media_type, 1, 66, width, height,
        par_n, par_d);
  }
  else if (max_framerate > 0)
  {
    add_one_resolution_inner (extra_low_caps, media_type, 1, 66, width, height,
        par_n, par_d);
  }
}


GstCaps *
caps_from_bitrate (const gchar *media_type, guint bitrate)
{
  GstCaps *caps = gst_caps_new_empty ();
  GstCaps *lower_caps = gst_caps_new_empty ();
  GstCaps *extra_low_caps = gst_caps_new_empty ();
  guint max_pixels_per_second = bitrate * H264_MAX_PIXELS_PER_BIT;
  gint i;

  /* At least one FPS at a very low res */
  max_pixels_per_second = MAX (max_pixels_per_second, 128 * 96);

  for (i = 0; one_on_one_resolutions[i].width > 1; i++)
    add_one_resolution (media_type, caps, lower_caps, extra_low_caps,
        max_pixels_per_second,
        one_on_one_resolutions[i].width,
        one_on_one_resolutions[i].height, 1, 1);

  for (i = 0; twelve_on_eleven_resolutions[i].width > 1; i++)
    add_one_resolution (media_type, caps, lower_caps, extra_low_caps,
        twelve_on_eleven_resolutions[i].width,
        twelve_on_eleven_resolutions[i].height,
        max_pixels_per_second, 12, 11);

  gst_caps_append (caps, lower_caps);
  if (gst_caps_is_empty (caps))
    gst_caps_append (caps, extra_low_caps);
  else
    gst_caps_unref (extra_low_caps);

  return caps;
}


static GstCaps *
fs_rtp_bitrate_adapter_getcaps (FsRtpBitrateAdapter *self, GstPad *pad,
    GstCaps *filter)
{
  GstPad *otherpad;
  GstCaps *peer_caps;
  GstCaps *result;
  guint bitrate;
  guint i;

  if (pad == self->srcpad)
    otherpad = self->sinkpad;
  else
    otherpad = self->srcpad;

  peer_caps = gst_pad_peer_query_caps (otherpad, filter);

  if (gst_caps_get_size (peer_caps) == 0)
    return peer_caps;

  GST_OBJECT_LOCK (self);
  bitrate = self->bitrate;
  if (pad == self->sinkpad)
    self->last_bitrate = self->bitrate;
  GST_OBJECT_UNLOCK (self);

  if (bitrate == G_MAXUINT)
    return peer_caps;

  result = gst_caps_new_empty ();

  for (i = 0; i < gst_caps_get_size (peer_caps); i++)
  {
    GstStructure *s = gst_caps_get_structure (peer_caps, i);

    if (g_str_has_prefix (gst_structure_get_name (s), "video/"))
    {
      GstCaps *rated_caps = caps_from_bitrate (gst_structure_get_name (s),
          bitrate);
      GstCaps *copy = gst_caps_copy_nth (peer_caps, i);

      gst_caps_set_features (rated_caps, 0,
          gst_caps_features_copy (gst_caps_get_features (peer_caps, i)));

      gst_caps_append (result, gst_caps_intersect (rated_caps, copy));
      gst_caps_unref (copy);
      gst_caps_unref (rated_caps);
    }
    else
    {
      gst_caps_append (result, gst_caps_copy_nth (peer_caps, i));
    }
  }

  return result;
}

static gboolean
fs_rtp_bitrate_adapter_query (GstPad *pad, GstObject *parent, GstQuery *query)
{
  FsRtpBitrateAdapter *self = FS_RTP_BITRATE_ADAPTER (parent);
  gboolean res;

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CAPS:
    {
      GstCaps *caps, *filter;

      gst_query_parse_caps (query, &filter);
      caps = fs_rtp_bitrate_adapter_getcaps (self, pad, filter);
      gst_query_set_caps_result (query, caps);
      gst_caps_unref (caps);
      res = TRUE;
      break;
    }
    default:
      res = gst_pad_query_default (pad, parent, query);
      break;
  }

  return res;
}

static GstFlowReturn
fs_rtp_bitrate_adapter_chain (GstPad *pad, GstObject *parent,
    GstBuffer *buffer)
{
  FsRtpBitrateAdapter *self = FS_RTP_BITRATE_ADAPTER (parent);
  GstFlowReturn ret;

  if (!self)
    return GST_FLOW_NOT_LINKED;

  ret = gst_pad_push (self->srcpad, buffer);

  return ret;
}


static guint
fs_rtp_bitrate_adapter_get_bitrate_locked (FsRtpBitrateAdapter *self)
{
  gdouble mean = 0;
  guint count = 0;
  gdouble S = 0;
  GList *item;
  gdouble stddev;

  for (item = self->bitrate_history.head; item ;item = item->next) {
    struct BitratePoint *bp = item->data;
    gdouble delta;

    count++;
    delta = bp->bitrate - mean;
    mean = mean + delta/count;
    S = S + delta * (bp->bitrate - mean);
  }

  if (count == 0)
    return G_MAXUINT;

  g_assert (S >= 0);
  stddev = sqrt (S/count);

  if (mean > stddev)
    return (guint) (mean - stddev);
  else
    return G_MAXUINT;
}

static void
fs_rtp_bitrate_adapter_updated_unlock (FsRtpBitrateAdapter *self)
{
  gboolean changed = FALSE;

  self->bitrate = fs_rtp_bitrate_adapter_get_bitrate_locked (self);

  GST_DEBUG ("Computed average lower bitrate: %u", self->bitrate);
  if (self->bitrate != G_MAXUINT &&
      (self->bitrate > self->last_bitrate * 1.1 ||
          self->bitrate < self->last_bitrate * 0.9))
  {
    self->last_bitrate = self->bitrate;
    changed = TRUE;
  }
  GST_OBJECT_UNLOCK (self);

  if (changed)
    gst_pad_push_event (self->sinkpad, gst_event_new_reconfigure ());
}

static void
fs_rtp_bitrate_adapter_cleanup_locked (FsRtpBitrateAdapter *self,
    GstClockTime now)
{
  for (;;)
  {
    struct BitratePoint *bp = g_queue_peek_head (&self->bitrate_history);

    if (bp && (bp->timestamp < now - self->interval ||
            (GST_STATE (self) != GST_STATE_PLAYING &&
                g_queue_get_length (&self->bitrate_history) > 1)))
    {
      g_queue_pop_head (&self->bitrate_history);
      bitrate_point_free (bp);
    }
    else
    {
      break;
    }
  }
}

static gboolean
clock_callback (GstClock *clock, GstClockTime now, GstClockID clockid,
    gpointer user_data)
{
  FsRtpBitrateAdapter *self = user_data;

  GST_OBJECT_LOCK (self);
  if (self->clockid == clockid)
  {
    gst_clock_id_unref (self->clockid);
  }
  else
  {
    GST_OBJECT_UNLOCK (self);
    return TRUE;
  }
  self->clockid = NULL;

  fs_rtp_bitrate_adapter_updated_unlock (self);


  return TRUE;
}

static gboolean
fs_rtp_bitrate_adapter_add_bitrate_locked (FsRtpBitrateAdapter *self,
    guint bitrate)
{
  GstClockTime now = gst_clock_get_time (self->system_clock);
  gboolean first = FALSE;

  g_queue_push_tail (&self->bitrate_history, bitrate_point_new (now, bitrate));

  first = (g_queue_get_length (&self->bitrate_history) == 1);

  fs_rtp_bitrate_adapter_cleanup_locked (self, now);

  if (!self->clockid && GST_STATE (self) == GST_STATE_PLAYING)
  {
    self->clockid = gst_clock_new_single_shot_id (self->system_clock,
        now + self->interval);
    gst_clock_id_wait_async (self->clockid,
        clock_callback, gst_object_ref (self), gst_object_unref);
  }

  return first;
}


static void
fs_rtp_bitrate_adapter_set_property (GObject *object,
    guint prop_id,
    const GValue *value,
    GParamSpec *pspec)
{
  FsRtpBitrateAdapter *self = FS_RTP_BITRATE_ADAPTER (object);
  gboolean first = FALSE;

  GST_OBJECT_LOCK (self);
  switch (prop_id)
  {
    case PROP_BITRATE:
      first = fs_rtp_bitrate_adapter_add_bitrate_locked (self,
          g_value_get_uint (value));
      break;
    case PROP_INTERVAL:
      self->interval = g_value_get_uint64 (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }

  if (first)
    fs_rtp_bitrate_adapter_updated_unlock (self);
  else
    GST_OBJECT_UNLOCK (self);
}


static GstStateChangeReturn
fs_rtp_bitrate_adapter_change_state (GstElement *element,
    GstStateChange transition)
{
  FsRtpBitrateAdapter *self = FS_RTP_BITRATE_ADAPTER (element);
  GstStateChangeReturn result;

  switch (transition) {
    case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
      GST_OBJECT_LOCK (self);
      if (self->clockid)
      {
        gst_clock_id_unschedule (self->clockid);
        gst_clock_id_unref (self->clockid);
      }
      self->clockid = NULL;
      GST_OBJECT_UNLOCK (self);

      break;
    case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
      GST_OBJECT_LOCK (self);
      if (g_queue_get_length (&self->bitrate_history))
        fs_rtp_bitrate_adapter_updated_unlock (self);
      else
        GST_OBJECT_UNLOCK (self);
      break;
    default:
      break;
  }

  if ((result =
          GST_ELEMENT_CLASS (fs_rtp_bitrate_adapter_parent_class)->change_state
          (element, transition)) == GST_STATE_CHANGE_FAILURE)
    goto failure;

  switch (transition) {
    case GST_STATE_CHANGE_PAUSED_TO_READY:
      self->last_bitrate = G_MAXUINT;
      g_queue_foreach (&self->bitrate_history, (GFunc) bitrate_point_free,
          NULL);
      g_queue_clear(&self->bitrate_history);
      break;
    default:
      break;
  }


  return result;

 failure:
  {
    GST_ERROR_OBJECT (element, "parent failed state change");
    return result;
  }
}

GstElement *
fs_rtp_bitrate_adapter_new (void)
{
  return g_object_new (FS_TYPE_RTP_BITRATE_ADAPTER, NULL);
}