From 7c442abce73c462e7724a64b3ce037bcb8e9093e Mon Sep 17 00:00:00 2001 From: Collin Date: Tue, 1 Oct 2019 15:17:05 -0400 Subject: [WIP] initial fix of socket streaming by guessing playback time (#3057) * initial fix of socket streaming by guessing playback time * isolate custom timer logic to socket audio streaming * add half second latency compensation * add mocks for changed and new methods * make DataSizeToMilliseconds const method * fix unit tests * update server_type checks * address review comments * fix namespace of ServiceType --- .../include/application_manager/application.h | 4 +- .../include/application_manager/application_impl.h | 3 +- .../application_manager/src/application_impl.cc | 12 +++-- .../include/application_manager/mock_application.h | 5 +- .../include/media_manager/media_manager.h | 8 ++++ .../test/media_manager/mock_media_manager.h | 1 + .../include/media_manager/media_manager_impl.h | 9 ++++ .../media_manager/src/media_manager_impl.cc | 54 ++++++++++++++++++++-- .../media_manager/test/media_manager_impl_test.cc | 8 +++- 9 files changed, 91 insertions(+), 13 deletions(-) diff --git a/src/components/application_manager/include/application_manager/application.h b/src/components/application_manager/include/application_manager/application.h index 8e78874cdc..e9dcf2bcb0 100644 --- a/src/components/application_manager/include/application_manager/application.h +++ b/src/components/application_manager/include/application_manager/application.h @@ -656,8 +656,10 @@ class Application : public virtual InitialApplicationData, /** * @brief Wakes up streaming process for application * @param service_type Type of streaming service + * @param timer_len The amount of time in ms the timer will wait */ - virtual void WakeUpStreaming(protocol_handler::ServiceType service_type) = 0; + virtual void WakeUpStreaming(protocol_handler::ServiceType service_type, + uint32_t timer_len = 0) = 0; virtual bool is_voice_communication_supported() const = 0; virtual void set_voice_communication_supported( diff --git a/src/components/application_manager/include/application_manager/application_impl.h b/src/components/application_manager/include/application_manager/application_impl.h index 23dc41ecd7..27b6c8123e 100644 --- a/src/components/application_manager/include/application_manager/application_impl.h +++ b/src/components/application_manager/include/application_manager/application_impl.h @@ -139,7 +139,8 @@ class ApplicationImpl : public virtual Application, void StopStreamingForce(protocol_handler::ServiceType service_type); void StopStreaming(protocol_handler::ServiceType service_type); void SuspendStreaming(protocol_handler::ServiceType service_type); - void WakeUpStreaming(protocol_handler::ServiceType service_type); + void WakeUpStreaming(protocol_handler::ServiceType service_type, + uint32_t timer_len = 0); virtual bool is_voice_communication_supported() const; virtual void set_voice_communication_supported( diff --git a/src/components/application_manager/src/application_impl.cc b/src/components/application_manager/src/application_impl.cc index 683054618b..2b328ae9a1 100644 --- a/src/components/application_manager/src/application_impl.cc +++ b/src/components/application_manager/src/application_impl.cc @@ -634,7 +634,7 @@ void ApplicationImpl::SuspendStreaming( } void ApplicationImpl::WakeUpStreaming( - protocol_handler::ServiceType service_type) { + protocol_handler::ServiceType service_type, uint32_t timer_len) { using namespace protocol_handler; LOG4CXX_AUTO_TRACE(logger_); @@ -650,8 +650,9 @@ void ApplicationImpl::WakeUpStreaming( ServiceType::kMobileNav, true, application_manager_); video_streaming_suspended_ = false; } - video_stream_suspend_timer_.Start(video_stream_suspend_timeout_, - timer::kPeriodic); + video_stream_suspend_timer_.Start( + timer_len == 0 ? video_stream_suspend_timeout_ : timer_len, + timer::kPeriodic); } else if (ServiceType::kAudio == service_type) { sync_primitives::AutoLock lock(audio_streaming_suspended_lock_); if (audio_streaming_suspended_) { @@ -660,8 +661,9 @@ void ApplicationImpl::WakeUpStreaming( ServiceType::kAudio, true, application_manager_); audio_streaming_suspended_ = false; } - audio_stream_suspend_timer_.Start(audio_stream_suspend_timeout_, - timer::kPeriodic); + audio_stream_suspend_timer_.Start( + timer_len == 0 ? audio_stream_suspend_timeout_ : timer_len, + timer::kPeriodic); } } diff --git a/src/components/application_manager/test/include/application_manager/mock_application.h b/src/components/application_manager/test/include/application_manager/mock_application.h index e3617b46c3..362dda4efb 100644 --- a/src/components/application_manager/test/include/application_manager/mock_application.h +++ b/src/components/application_manager/test/include/application_manager/mock_application.h @@ -86,8 +86,9 @@ class MockApplication : public ::application_manager::Application { void(protocol_handler::ServiceType service_type)); MOCK_METHOD1(SuspendStreaming, void(protocol_handler::ServiceType service_type)); - MOCK_METHOD1(WakeUpStreaming, - void(protocol_handler::ServiceType service_type)); + MOCK_METHOD2(WakeUpStreaming, + void(protocol_handler::ServiceType service_type, + uint32_t timer_len)); MOCK_CONST_METHOD0(is_voice_communication_supported, bool()); MOCK_METHOD1(set_voice_communication_supported, void(bool is_voice_communication_supported)); diff --git a/src/components/include/media_manager/media_manager.h b/src/components/include/media_manager/media_manager.h index 0998899bc5..123e11f2f0 100644 --- a/src/components/include/media_manager/media_manager.h +++ b/src/components/include/media_manager/media_manager.h @@ -60,6 +60,14 @@ class MediaManager { */ virtual const MediaManagerSettings& settings() const = 0; + /** + * \brief Convert an amount of audio bytes to an estimated time in ms + * \param data_size number of bytes to be played + * \return milliseconds required to play many bytes with + * the current pcm stream capabilities + */ + virtual uint32_t DataSizeToMilliseconds(uint64_t data_size) const = 0; + virtual ~MediaManager() {} }; diff --git a/src/components/include/test/media_manager/mock_media_manager.h b/src/components/include/test/media_manager/mock_media_manager.h index b58cfab5d7..aa0baa5682 100644 --- a/src/components/include/test/media_manager/mock_media_manager.h +++ b/src/components/include/test/media_manager/mock_media_manager.h @@ -57,6 +57,7 @@ class MockMediaManager : public media_manager::MediaManager { MOCK_METHOD2(FramesProcessed, void(int32_t application_key, int32_t frame_number)); MOCK_CONST_METHOD0(settings, const media_manager::MediaManagerSettings&()); + MOCK_CONST_METHOD1(DataSizeToMilliseconds, uint32_t(uint64_t data_size)); }; } // namespace media_manager_test diff --git a/src/components/media_manager/include/media_manager/media_manager_impl.h b/src/components/media_manager/include/media_manager/media_manager_impl.h index 30b05d5c6b..408fa12655 100644 --- a/src/components/media_manager/include/media_manager/media_manager_impl.h +++ b/src/components/media_manager/include/media_manager/media_manager_impl.h @@ -33,6 +33,7 @@ #ifndef SRC_COMPONENTS_MEDIA_MANAGER_INCLUDE_MEDIA_MANAGER_MEDIA_MANAGER_IMPL_H_ #define SRC_COMPONENTS_MEDIA_MANAGER_INCLUDE_MEDIA_MANAGER_MEDIA_MANAGER_IMPL_H_ +#include #include #include #include "media_manager/media_adapter_impl.h" @@ -81,6 +82,8 @@ class MediaManagerImpl : public MediaManager, virtual const MediaManagerSettings& settings() const OVERRIDE; + virtual uint32_t DataSizeToMilliseconds(uint64_t data_size) const OVERRIDE; + #ifdef BUILD_TESTS void set_mock_a2dp_player(MediaAdapter* media_adapter); void set_mock_mic_listener(MediaListenerPtr media_listener); @@ -106,6 +109,12 @@ class MediaManagerImpl : public MediaManager, std::map streamer_; std::map streamer_listener_; + uint32_t bits_per_sample_; + uint32_t sampling_rate_; + uint64_t stream_data_size_; + std::chrono::time_point + socket_audio_stream_start_time_; + application_manager::ApplicationManager& application_manager_; private: diff --git a/src/components/media_manager/src/media_manager_impl.cc b/src/components/media_manager/src/media_manager_impl.cc index bc93f2f764..1f09f67ace 100644 --- a/src/components/media_manager/src/media_manager_impl.cc +++ b/src/components/media_manager/src/media_manager_impl.cc @@ -35,6 +35,7 @@ #include "application_manager/application_impl.h" #include "application_manager/application_manager.h" #include "application_manager/message_helper.h" +#include "application_manager/smart_object_keys.h" #include "media_manager/audio/from_mic_recorder_listener.h" #include "media_manager/streamer_listener.h" #include "protocol_handler/protocol_handler.h" @@ -64,6 +65,9 @@ MediaManagerImpl::MediaManagerImpl( , protocol_handler_(NULL) , a2dp_player_(NULL) , from_mic_recorder_(NULL) + , bits_per_sample_(16) + , sampling_rate_(16000) + , stream_data_size_(0ull) , application_manager_(application_manager) { Init(); } @@ -160,6 +164,23 @@ void MediaManagerImpl::Init() { streamer_[ServiceType::kAudio]->AddListener( streamer_listener_[ServiceType::kAudio]); } + + if (application_manager_.hmi_capabilities().pcm_stream_capabilities()) { + const auto pcm_caps = + application_manager_.hmi_capabilities().pcm_stream_capabilities(); + + if (pcm_caps->keyExists(application_manager::strings::bits_per_sample)) { + bits_per_sample_ = + pcm_caps->getElement(application_manager::strings::bits_per_sample) + .asUInt(); + } + + if (pcm_caps->keyExists(application_manager::strings::sampling_rate)) { + sampling_rate_ = + pcm_caps->getElement(application_manager::strings::sampling_rate) + .asUInt(); + } + } } void MediaManagerImpl::PlayA2DPSource(int32_t application_key) { @@ -254,6 +275,11 @@ void MediaManagerImpl::StartStreaming( int32_t application_key, protocol_handler::ServiceType service_type) { LOG4CXX_AUTO_TRACE(logger_); + if (protocol_handler::ServiceType::kAudio == service_type && + "socket" == settings().audio_server_type()) { + socket_audio_stream_start_time_ = std::chrono::system_clock::now(); + } + if (streamer_[service_type]) { streamer_[service_type]->StartActivity(application_key); } @@ -298,7 +324,21 @@ void MediaManagerImpl::OnMessageReceived( ApplicationSharedPtr app = application_manager_.application(streaming_app_id); if (app) { - app->WakeUpStreaming(service_type); + if (ServiceType::kAudio == service_type && + "socket" == settings().audio_server_type()) { + stream_data_size_ += message->data_size(); + uint32_t ms_for_all_data = DataSizeToMilliseconds(stream_data_size_); + uint32_t ms_since_stream_start = + std::chrono::duration_cast( + std::chrono::system_clock::now() - + socket_audio_stream_start_time_) + .count(); + uint32_t ms_stream_remaining = ms_for_all_data - ms_since_stream_start; + + app->WakeUpStreaming(service_type, ms_stream_remaining); + } else { + app->WakeUpStreaming(service_type); + } streamer_[service_type]->SendData(streaming_app_id, message); } } @@ -321,7 +361,8 @@ void MediaManagerImpl::FramesProcessed(int32_t application_key, auto video_stream = std::dynamic_pointer_cast( streamer_[protocol_handler::ServiceType::kMobileNav]); - if (audio_stream.use_count() != 0) { + if (audio_stream.use_count() != 0 && + "pipe" == settings().audio_server_type()) { size_t audio_queue_size = audio_stream->GetMsgQueueSize(); LOG4CXX_DEBUG(logger_, "# Messages in audio queue = " << audio_queue_size); @@ -330,7 +371,8 @@ void MediaManagerImpl::FramesProcessed(int32_t application_key, } } - if (video_stream.use_count() != 0) { + if (video_stream.use_count() != 0 && + "pipe" == settings().video_server_type()) { size_t video_queue_size = video_stream->GetMsgQueueSize(); LOG4CXX_DEBUG(logger_, "# Messages in video queue = " << video_queue_size); @@ -345,4 +387,10 @@ const MediaManagerSettings& MediaManagerImpl::settings() const { return settings_; } +uint32_t MediaManagerImpl::DataSizeToMilliseconds(uint64_t data_size) const { + constexpr uint16_t latency_compensation = 500; + return 1000 * data_size / (sampling_rate_ * bits_per_sample_ / 8) + + latency_compensation; +} + } // namespace media_manager diff --git a/src/components/media_manager/test/media_manager_impl_test.cc b/src/components/media_manager/test/media_manager_impl_test.cc index 3c537f2337..ff35ac7dbb 100644 --- a/src/components/media_manager/test/media_manager_impl_test.cc +++ b/src/components/media_manager/test/media_manager_impl_test.cc @@ -35,6 +35,7 @@ #include "application_manager/message.h" #include "application_manager/mock_application.h" #include "application_manager/mock_application_manager.h" +#include "application_manager/mock_hmi_capabilities.h" #include "application_manager/resumption/resume_ctrl.h" #include "application_manager/state_controller.h" #include "gmock/gmock.h" @@ -109,6 +110,10 @@ class MediaManagerImplTest : public ::testing::Test { .WillByDefault(ReturnRef(kDefaultValue)); ON_CALL(mock_media_manager_settings_, audio_server_type()) .WillByDefault(ReturnRef(kDefaultValue)); + ON_CALL(mock_hmi_capabilities_, pcm_stream_capabilities()) + .WillByDefault(Return(nullptr)); + ON_CALL(app_mngr_, hmi_capabilities()) + .WillByDefault(ReturnRef(mock_hmi_capabilities_)); mock_app_ = std::make_shared(); media_manager_impl_.reset( new MediaManagerImpl(app_mngr_, mock_media_manager_settings_)); @@ -176,7 +181,7 @@ class MediaManagerImplTest : public ::testing::Test { .WillOnce(Return(true)); EXPECT_CALL(app_mngr_, application(kConnectionKey)) .WillOnce(Return(mock_app_)); - EXPECT_CALL(*mock_app_, WakeUpStreaming(service_type)); + EXPECT_CALL(*mock_app_, WakeUpStreaming(service_type, 0ull)); MockMediaAdapterImplPtr mock_media_streamer = std::make_shared(); media_manager_impl_->set_mock_streamer(service_type, mock_media_streamer); @@ -206,6 +211,7 @@ class MediaManagerImplTest : public ::testing::Test { const ::testing::NiceMock mock_media_manager_settings_; std::shared_ptr media_manager_impl_; + application_manager_test::MockHMICapabilities mock_hmi_capabilities_; }; TEST_F(MediaManagerImplTest, -- cgit v1.2.1