Reduce choppy sound on audio playback if audio sink is almost empty

If audio sink buffer is almost empty, that happens on pause/play,
the sound pretty often has little gaps. The solution is using some
little samples compensation (slows down the playback on 1% for
1-2sec what allows to increase buffer loadig from 0-10% to 20%).

In other words, the patch implements slight "soft" audio samples
compensation in order to avoid having empty adio sink buffer.

Pick-to: 6.5
Change-Id: I6b963996eab8f8b8f610fcd4a566405aec3d13cf
Reviewed-by: Lars Knoll <lars@knoll.priv.no>

6 files changed, 84 insertions, 7 deletions

diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp
index 7afd379d4..e44736c91 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp
@@ -5,20 +5,29 @@
 #include "qaudiosink.h"
 #include "qaudiooutput.h"
 #include "private/qplatformaudiooutput_p.h"
+#include <QtCore/qloggingcategory.h>
 
 #include "qffmpegresampler_p.h"
 #include "qffmpegmediaformatinfo_p.h"
 
 QT_BEGIN_NAMESPACE
 
+static Q_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audiorenderer");
+
 namespace QFFmpeg {
 
-constexpr std::chrono::microseconds audioSinkBufferSize(100000);
+using namespace std::chrono_literals;
+
+namespace {
+constexpr auto AudioSinkBufferTime = 100000us;
+constexpr auto MinDesiredBufferTime = AudioSinkBufferTime / 10;
+
+// actual playback rate chang during the soft compensation
+constexpr qreal CompensationAngleFactor = 0.01;
+} // namespace
 
 AudioRenderer::AudioRenderer(const TimeController &tc, QAudioOutput *output)
-    : Renderer(tc,
-               audioSinkBufferSize / 2 /*Ensures kind of "spring" in order to avoid chopy sound*/),
-      m_output(output)
+    : Renderer(tc, MinDesiredBufferTime), m_output(output)
 {
     if (output) {
         // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294
@@ -56,6 +65,7 @@ Renderer::RenderingResult AudioRenderer::renderInternal(Frame frame)
         if (!frame.isValid())
             return {};
 
+        updateSampleCompensation(frame);
         m_bufferedData = m_resampler->resample(frame.avFrame());
         m_bufferWritten = 0;
     }
@@ -106,6 +116,7 @@ void AudioRenderer::initResempler(const Codec *codec)
 
 void AudioRenderer::freeOutput()
 {
+    qCDebug(qLcAudioRenderer) << "Free audio output";
     if (m_sink) {
         m_sink->reset();
 
@@ -141,7 +152,7 @@ void AudioRenderer::updateOutput(const Codec *codec)
     if (!m_sink) {
         m_sink = std::make_unique<QAudioSink>(m_output->device(), m_format);
         updateVolume();
-        m_sink->setBufferSize(m_format.bytesForDuration(audioSinkBufferSize.count()));
+        m_sink->setBufferSize(m_format.bytesForDuration(AudioSinkBufferTime.count()));
         m_ioDevice = m_sink->start();
     }
 
@@ -150,6 +161,48 @@ void AudioRenderer::updateOutput(const Codec *codec)
     }
 }
 
+void AudioRenderer::updateSampleCompensation(const Frame &currentFrame)
+{
+    // Currently we use "soft" compensation with a positive delta
+    // for slight increasing of the buffer loading if it's too low
+    // in order to avoid choppy sound. If the bufer loading is too low,
+    // QAudioSink sometimes utilizes all written data earlier than new data delivered,
+    // that produces sound clicks (most hearable on Windows).
+    //
+    // TODO:
+    // 1. Probably, use "hard" compensation (inject silence) on the start and after pause
+    // 2. Probably, use "soft" compensation with a negative delta for decreasing buffer loading.
+    //    Currently, we use renderers synchronizations, but the suggested approach might imrove
+    //    the sound rendering and avoid changing of the current rendering position.
+
+    Q_ASSERT(m_sink);
+    Q_ASSERT(m_resampler);
+    Q_ASSERT(currentFrame.isValid());
+
+    const auto loadBufferTime = AudioSinkBufferTime
+            * qMax(m_sink->bufferSize() - m_sink->bytesFree(), 0) / m_sink->bufferSize();
+
+    constexpr auto frameDelayThreshold = MinDesiredBufferTime / 2;
+    const bool positiveCompensationNeeded = loadBufferTime < MinDesiredBufferTime
+            && !m_resampler->isSampleCompensationActive()
+            && frameDelay(currentFrame) < frameDelayThreshold;
+
+    if (positiveCompensationNeeded) {
+        constexpr auto targetBufferTime = MinDesiredBufferTime * 2;
+        const auto delta = m_format.sampleRate() * (targetBufferTime - loadBufferTime) / 1s;
+        const auto interval = delta / CompensationAngleFactor;
+
+        qCDebug(qLcAudioRenderer) << "Enable audio sample speed up compensation. Delta:" << delta
+                                  << "Interval:" << interval
+                                  << "SampleRate:" << m_format.sampleRate()
+                                  << "SinkLoadTime(us):" << loadBufferTime.count()
+                                  << "SamplesProcessed:" << m_resampler->samplesProcessed();
+
+        m_resampler->setSampleCompensation(static_cast<qint32>(delta),
+                                           static_cast<quint32>(interval));
+    }
+}
+
 } // namespace QFFmpeg
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h
index 270a4abf8..b5aef8a71 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h
@@ -52,6 +52,8 @@ protected:
 
     void updateVolume();
 
+    void updateSampleCompensation(const Frame &currentFrame);
+
 private:
     QPointer<QAudioOutput> m_output;
     std::unique_ptr<QAudioSink> m_sink;
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp
index 488bf745c..735f0b12e 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp
@@ -164,6 +164,12 @@ void Renderer::doNextStep()
 
     scheduleNextStep(false);
 }
+
+std::chrono::microseconds Renderer::frameDelay(const Frame &frame) const
+{
+    return std::chrono::duration_cast<std::chrono::microseconds>(
+            TimeController::Clock::now() - m_timeController.timeFromPosition(frame.absolutePts()));
+}
 } // namespace QFFmpeg
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h
index 8949213a4..e0ff9072f 100644
--- a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h
+++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h
@@ -75,6 +75,8 @@ protected:
 
     float playbackRate() const;
 
+    std::chrono::microseconds frameDelay(const Frame &frame) const;
+
 private:
     void doNextStep() override;
 
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp b/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp
index 9d5882504..ecb650f9d 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp
@@ -85,11 +85,22 @@ QAudioBuffer Resampler::resample(const AVFrame *frame)
     m_samplesProcessed += out_samples;
 
     qCDebug(qLcResampler) << "    new frame" << startTime << "in_samples" << frame->nb_samples << out_samples << outSamples;
-    QAudioBuffer buffer(samples, m_outputFormat, startTime);
-    return buffer;
+    return QAudioBuffer(samples, m_outputFormat, startTime);
 }
 
+void Resampler::setSampleCompensation(qint32 delta, quint32 distance)
+{
+    const int res = swr_set_compensation(resampler, delta, static_cast<int>(distance));
+    if (res < 0)
+        qCWarning(qLcResampler) << "swr_set_compensation fail:" << res;
+    else
+        m_endCompensationSample = m_samplesProcessed + distance;
+}
 
+bool Resampler::isSampleCompensationActive() const
+{
+    return m_samplesProcessed < m_endCompensationSample;
+}
 }
 
 QT_END_NAMESPACE
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h b/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h
index 2f23c3c81..2ad008cf1 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h
+++ b/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h
@@ -32,11 +32,14 @@ public:
 
     QAudioBuffer resample(const AVFrame *frame);
     qint64 samplesProcessed() const { return m_samplesProcessed; }
+    void setSampleCompensation(qint32 delta, quint32 distance);
+    bool isSampleCompensationActive() const;
 
 private:
     QAudioFormat m_outputFormat;
     SwrContext *resampler = nullptr;
     qint64 m_samplesProcessed = 0;
+    qint64 m_endCompensationSample = std::numeric_limits<qint64>::min();
 };
 
 }