diff options
Diffstat (limited to 'chromium/chrome/renderer/media')
27 files changed, 241 insertions, 2702 deletions
diff --git a/chromium/chrome/renderer/media/OWNERS b/chromium/chrome/renderer/media/OWNERS index e5a3f968d1c..3de875bc8e3 100644 --- a/chromium/chrome/renderer/media/OWNERS +++ b/chromium/chrome/renderer/media/OWNERS @@ -2,10 +2,6 @@ file://media/OWNERS sergeyu@chromium.org tommi@chromium.org -# For Cast-related changes. -per-file cast_*=miu@chromium.org -per-file cast_*=mfoltz@chromium.org - # FlashEmbedRewrite per-file flash_embed_rewrite*=mlamouri@chromium.org diff --git a/chromium/chrome/renderer/media/cast_ipc_dispatcher.cc b/chromium/chrome/renderer/media/cast_ipc_dispatcher.cc deleted file mode 100644 index 3025cc4fbc7..00000000000 --- a/chromium/chrome/renderer/media/cast_ipc_dispatcher.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_ipc_dispatcher.h" - -#include "base/single_thread_task_runner.h" -#include "chrome/common/cast_messages.h" -#include "chrome/renderer/media/cast_transport_ipc.h" -#include "ipc/ipc_message_macros.h" - -CastIPCDispatcher* CastIPCDispatcher::global_instance_ = NULL; - -CastIPCDispatcher::CastIPCDispatcher( - const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner) - : sender_(NULL), - io_task_runner_(io_task_runner) { - DCHECK(io_task_runner_.get()); - DCHECK(!global_instance_); -} - -CastIPCDispatcher::~CastIPCDispatcher() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK(!global_instance_); -} - -CastIPCDispatcher* CastIPCDispatcher::Get() { - return global_instance_; -} - -void CastIPCDispatcher::Send(IPC::Message* message) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - if (sender_) { - sender_->Send(message); - } else { - delete message; - } -} - -int32_t CastIPCDispatcher::AddSender(CastTransportIPC* sender) { - return id_map_.Add(sender); -} - -void CastIPCDispatcher::RemoveSender(int32_t channel_id) { - return id_map_.Remove(channel_id); -} - -bool CastIPCDispatcher::OnMessageReceived(const IPC::Message& message) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - bool handled = true; - IPC_BEGIN_MESSAGE_MAP(CastIPCDispatcher, message) - IPC_MESSAGE_HANDLER(CastMsg_NotifyStatusChange, OnNotifyStatusChange) - IPC_MESSAGE_HANDLER(CastMsg_RawEvents, OnRawEvents) - IPC_MESSAGE_HANDLER(CastMsg_Rtt, OnRtt) - IPC_MESSAGE_HANDLER(CastMsg_RtcpCastMessage, OnRtcpCastMessage) - IPC_MESSAGE_HANDLER(CastMsg_Pli, OnReceivedPli); - IPC_MESSAGE_HANDLER(CastMsg_ReceivedPacket, OnReceivedPacket) - IPC_MESSAGE_UNHANDLED(handled = false) - IPC_END_MESSAGE_MAP() - return handled; -} - -void CastIPCDispatcher::OnFilterAdded(IPC::Channel* channel) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK(!global_instance_); - global_instance_ = this; - sender_ = channel; -} - -void CastIPCDispatcher::OnFilterRemoved() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK_EQ(this, global_instance_); - global_instance_ = NULL; - sender_ = NULL; -} - -void CastIPCDispatcher::OnChannelClosing() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK_EQ(this, global_instance_); -} - -void CastIPCDispatcher::OnNotifyStatusChange( - int32_t channel_id, - media::cast::CastTransportStatus status) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnNotifyStatusChange(status); - } else { - DVLOG(1) - << "CastIPCDispatcher::OnNotifystatusChange on non-existing channel."; - } -} - -void CastIPCDispatcher::OnRawEvents( - int32_t channel_id, - const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnRawEvents(packet_events, frame_events); - } else { - DVLOG(1) << "CastIPCDispatcher::OnRawEvents on non-existing channel."; - } -} - -void CastIPCDispatcher::OnRtt(int32_t channel_id, - uint32_t ssrc, - base::TimeDelta rtt) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnRtt(ssrc, rtt); - } else { - DVLOG(1) << "CastIPCDispatcher::OnRtt on non-existing channel."; - } -} - -void CastIPCDispatcher::OnRtcpCastMessage( - int32_t channel_id, - uint32_t ssrc, - const media::cast::RtcpCastMessage& cast_message) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnRtcpCastMessage(ssrc, cast_message); - } else { - DVLOG(1) << "CastIPCDispatcher::OnRtt on non-existing channel."; - } -} - -void CastIPCDispatcher::OnReceivedPli(int32_t channel_id, int32_t ssrc) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnReceivedPli(ssrc); - } else { - DVLOG(1) << "CastIPCDispatcher::OnReceivedPli on non-existing " - "channel."; - } -} - -void CastIPCDispatcher::OnReceivedPacket(int32_t channel_id, - const media::cast::Packet& packet) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnReceivedPacket(packet); - } else { - DVLOG(1) << "CastIPCDispatcher::OnReceievdPacket on non-existing channel."; - } -} diff --git a/chromium/chrome/renderer/media/cast_ipc_dispatcher.h b/chromium/chrome/renderer/media/cast_ipc_dispatcher.h deleted file mode 100644 index 32f898fddb2..00000000000 --- a/chromium/chrome/renderer/media/cast_ipc_dispatcher.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_IPC_DISPATCHER_H_ -#define CHROME_RENDERER_MEDIA_CAST_IPC_DISPATCHER_H_ - -#include <stdint.h> - -#include "base/callback.h" -#include "base/containers/id_map.h" -#include "base/macros.h" -#include "base/threading/thread_task_runner_handle.h" -#include "ipc/ipc_channel_proxy.h" -#include "ipc/message_filter.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/net/cast_transport.h" - -class CastTransportIPC; - -// This dispatcher listens to incoming IPC messages and sends -// the call to the correct CastTransportIPC instance. -class CastIPCDispatcher : public IPC::MessageFilter { - public: - explicit CastIPCDispatcher( - const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner); - - static CastIPCDispatcher* Get(); - void Send(IPC::Message* message); - int32_t AddSender(CastTransportIPC* sender); - void RemoveSender(int32_t channel_id); - - // IPC::MessageFilter implementation - bool OnMessageReceived(const IPC::Message& message) override; - void OnFilterAdded(IPC::Channel* channel) override; - void OnFilterRemoved() override; - void OnChannelClosing() override; - - protected: - ~CastIPCDispatcher() override; - - private: - void OnNotifyStatusChange(int32_t channel_id, - media::cast::CastTransportStatus status); - void OnRtpStatistics(int32_t channel_id, - bool audio, - const media::cast::RtcpSenderInfo& sender_info, - base::TimeTicks time_sent, - uint32_t rtp_timestamp); - void OnRawEvents(int32_t channel_id, - const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events); - void OnRtt(int32_t channel_id, uint32_t ssrc, base::TimeDelta rtt); - void OnRtcpCastMessage(int32_t channel_id, - uint32_t ssrc, - const media::cast::RtcpCastMessage& cast_message); - void OnReceivedPli(int32_t channel_id, int32_t ssrc); - void OnReceivedPacket(int32_t channel_id, const media::cast::Packet& packet); - - static CastIPCDispatcher* global_instance_; - - // For IPC Send(); must only be accesed on |io_message_loop_|. - IPC::Sender* sender_; - - // Task runner on which IPC calls are driven. - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - - // A map of stream ids to delegates; must only be accessed on - // |io_message_loop_|. - base::IDMap<CastTransportIPC*> id_map_; - DISALLOW_COPY_AND_ASSIGN(CastIPCDispatcher); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_IPC_DISPATCHER_H_ diff --git a/chromium/chrome/renderer/media/cast_ipc_dispatcher_unittest.cc b/chromium/chrome/renderer/media/cast_ipc_dispatcher_unittest.cc deleted file mode 100644 index 7dd002ae427..00000000000 --- a/chromium/chrome/renderer/media/cast_ipc_dispatcher_unittest.cc +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_ipc_dispatcher.h" - -#include "base/test/simple_test_tick_clock.h" -#include "base/test/task_environment.h" -#include "base/threading/thread_task_runner_handle.h" -#include "chrome/common/cast_messages.h" -#include "ipc/ipc_message_macros.h" -#include "media/cast/logging/logging_defines.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace { - -class CastIPCDispatcherTest : public testing::Test { - public: - CastIPCDispatcherTest() { - dispatcher_ = new CastIPCDispatcher(base::ThreadTaskRunnerHandle::Get()); - } - - protected: - void FakeSend(const IPC::Message& message) { - EXPECT_TRUE(dispatcher_->OnMessageReceived(message)); - } - - scoped_refptr<CastIPCDispatcher> dispatcher_; - base::test::SingleThreadTaskEnvironment task_environment_{ - base::test::SingleThreadTaskEnvironment::MainThreadType::IO}; -}; - -TEST_F(CastIPCDispatcherTest, RawEvents) { - const int kChannelId = 17; - - media::cast::PacketEvent packet_event; - packet_event.rtp_timestamp = - media::cast::RtpTimeTicks().Expand(UINT32_C(100)); - packet_event.max_packet_id = 10; - packet_event.packet_id = 5; - packet_event.size = 512; - packet_event.timestamp = base::SimpleTestTickClock().NowTicks(); - packet_event.type = media::cast::PACKET_SENT_TO_NETWORK; - packet_event.media_type = media::cast::VIDEO_EVENT; - std::vector<media::cast::PacketEvent> packet_events; - packet_events.push_back(packet_event); - - media::cast::FrameEvent frame_event; - frame_event.rtp_timestamp = media::cast::RtpTimeTicks().Expand(UINT32_C(100)); - frame_event.frame_id = media::cast::FrameId::first() + 5; - frame_event.size = 512; - frame_event.timestamp = base::SimpleTestTickClock().NowTicks(); - frame_event.media_type = media::cast::VIDEO_EVENT; - std::vector<media::cast::FrameEvent> frame_events; - frame_events.push_back(frame_event); - - packet_events.push_back(packet_event); - CastMsg_RawEvents raw_events_msg(kChannelId, packet_events, - frame_events); - - FakeSend(raw_events_msg); -} - -} // namespace diff --git a/chromium/chrome/renderer/media/cast_receiver_audio_valve.cc b/chromium/chrome/renderer/media/cast_receiver_audio_valve.cc deleted file mode 100644 index c2d5b856727..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_audio_valve.cc +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_receiver_audio_valve.h" - -#include "base/bind.h" -#include "base/bind_helpers.h" -#include "media/base/audio_parameters.h" - -CastReceiverAudioValve::CastReceiverAudioValve( - const media::AudioParameters& params, - media::AudioCapturerSource::CaptureCallback* cb) - : cb_(cb), - fifo_(base::Bind(&CastReceiverAudioValve::DeliverRebufferedAudio, - base::Unretained(this))), - sample_rate_(params.sample_rate()) { - fifo_.Reset(params.frames_per_buffer()); -} - -CastReceiverAudioValve::~CastReceiverAudioValve() {} - -void CastReceiverAudioValve::DeliverDecodedAudio( - const media::AudioBus* audio_bus, - base::TimeTicks playout_time) { - current_playout_time_ = playout_time; - // The following will result in zero, one, or multiple synchronous calls to - // DeliverRebufferedAudio(). - fifo_.Push(*audio_bus); -} - -void CastReceiverAudioValve::DeliverRebufferedAudio( - const media::AudioBus& audio_bus, - int frame_delay) { - const base::TimeTicks playout_time = - current_playout_time_ + - base::TimeDelta::FromMicroseconds( - frame_delay * base::Time::kMicrosecondsPerSecond / sample_rate_); - - base::AutoLock lock(lock_); - if (cb_) { - cb_->Capture(&audio_bus, playout_time, 1.0 /* volume */, - false /* key_pressed */); - } -} - -void CastReceiverAudioValve::Stop() { - base::AutoLock lock(lock_); - cb_ = nullptr; -} diff --git a/chromium/chrome/renderer/media/cast_receiver_audio_valve.h b/chromium/chrome/renderer/media/cast_receiver_audio_valve.h deleted file mode 100644 index f6c09a74a26..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_audio_valve.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RECEIVER_AUDIO_VALVE_H_ -#define CHROME_RENDERER_MEDIA_CAST_RECEIVER_AUDIO_VALVE_H_ - -#include "base/synchronization/lock.h" -#include "base/time/time.h" -#include "media/base/audio_capturer_source.h" -#include "media/base/audio_push_fifo.h" - -namespace media { -class AudioBus; -} - -// Forwards calls to |cb| until Stop is called. If the client requested a -// different buffer size than that provided by the Cast Receiver, AudioPushFifo -// is used to rectify that. -// -// Thread-safe. -// All functions may block depending on contention. -class CastReceiverAudioValve : - public base::RefCountedThreadSafe<CastReceiverAudioValve> { - public: - CastReceiverAudioValve(const media::AudioParameters& params, - media::AudioCapturerSource::CaptureCallback* cb); - - // Called on an unknown thread to provide more decoded audio data from the - // Cast Receiver. - void DeliverDecodedAudio(const media::AudioBus* audio_bus, - base::TimeTicks playout_time); - - // When this returns, no more calls will be forwarded to |cb|. - void Stop(); - - private: - friend class base::RefCountedThreadSafe<CastReceiverAudioValve>; - - ~CastReceiverAudioValve(); - - // Called by AudioPushFifo zero or more times during the call to Capture(). - // Delivers audio data in the required buffer size to |cb_|. - void DeliverRebufferedAudio(const media::AudioBus& audio_bus, - int frame_delay); - - media::AudioCapturerSource::CaptureCallback* cb_; - base::Lock lock_; - - media::AudioPushFifo fifo_; - const int sample_rate_; - - // Used to pass the current playout time between DeliverDecodedAudio() and - // DeviliverRebufferedAudio(). - base::TimeTicks current_playout_time_; -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RECEIVER_AUDIO_VALVE_H_ diff --git a/chromium/chrome/renderer/media/cast_receiver_session.cc b/chromium/chrome/renderer/media/cast_receiver_session.cc deleted file mode 100644 index 659bf04ca4d..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session.cc +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_receiver_session.h" - -#include <memory> - -#include "base/bind.h" -#include "base/location.h" -#include "base/synchronization/waitable_event.h" -#include "base/threading/thread_task_runner_handle.h" -#include "chrome/renderer/media/cast_receiver_audio_valve.h" -#include "content/public/renderer/render_thread.h" -#include "media/base/audio_capturer_source.h" -#include "media/base/bind_to_current_loop.h" -#include "media/capture/video_capturer_source.h" -#include "third_party/blink/public/platform/web_media_stream.h" -#include "third_party/blink/public/platform/web_media_stream_source.h" -#include "third_party/blink/public/platform/web_media_stream_track.h" - -// This is a render thread object. -class CastReceiverSession::AudioCapturerSource : - public media::AudioCapturerSource { - public: - AudioCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session); - void Initialize(const media::AudioParameters& params, - CaptureCallback* callback) override; - void Start() override; - void Stop() override; - void SetVolume(double volume) override; - void SetAutomaticGainControl(bool enable) override; - void SetOutputDeviceForAec(const std::string& output_device_id) override; - - private: - ~AudioCapturerSource() override; - const scoped_refptr<CastReceiverSession> cast_receiver_session_; - scoped_refptr<CastReceiverAudioValve> audio_valve_; -}; - -// This is a render thread object. -class CastReceiverSession::VideoCapturerSource - : public media::VideoCapturerSource { - public: - explicit VideoCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session); - protected: - media::VideoCaptureFormats GetPreferredFormats() override; - void StartCapture(const media::VideoCaptureParams& params, - const VideoCaptureDeliverFrameCB& frame_callback, - const RunningCallback& running_callback) override; - void StopCapture() override; - private: - const scoped_refptr<CastReceiverSession> cast_receiver_session_; -}; - -CastReceiverSession::CastReceiverSession() - : delegate_(new CastReceiverSessionDelegate()), - io_task_runner_(content::RenderThread::Get()->GetIOTaskRunner()) {} - -CastReceiverSession::~CastReceiverSession() { - // We should always be able to delete the object on the IO thread. - CHECK(io_task_runner_->DeleteSoon(FROM_HERE, delegate_.release())); -} - -void CastReceiverSession::Start( - const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& capture_format, - const StartCB& start_callback, - const CastReceiverSessionDelegate::ErrorCallback& error_callback) { - audio_config_ = audio_config; - video_config_ = video_config; - format_ = capture_format; - io_task_runner_->PostTask( - FROM_HERE, base::BindOnce(&CastReceiverSessionDelegate::Start, - base::Unretained(delegate_.get()), audio_config, - video_config, local_endpoint, remote_endpoint, - std::move(options), format_, - media::BindToCurrentLoop(error_callback))); - scoped_refptr<media::AudioCapturerSource> audio( - new CastReceiverSession::AudioCapturerSource(this)); - std::unique_ptr<media::VideoCapturerSource> video( - new CastReceiverSession::VideoCapturerSource(this)); - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(start_callback, audio, std::move(video))); -} - -void CastReceiverSession::StartAudio( - scoped_refptr<CastReceiverAudioValve> audio_valve) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastReceiverSessionDelegate::StartAudio, - base::Unretained(delegate_.get()), audio_valve)); -} - -void CastReceiverSession::StartVideo( - blink::VideoCaptureDeliverFrameCB frame_callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastReceiverSessionDelegate::StartVideo, - base::Unretained(delegate_.get()), frame_callback)); -} - -void CastReceiverSession::StopVideo() { - io_task_runner_->PostTask( - FROM_HERE, base::BindOnce(&CastReceiverSessionDelegate::StopVideo, - base::Unretained(delegate_.get()))); -} - -CastReceiverSession::VideoCapturerSource::VideoCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session) - : cast_receiver_session_(cast_receiver_session) { -} - -media::VideoCaptureFormats -CastReceiverSession::VideoCapturerSource::GetPreferredFormats() { - media::VideoCaptureFormats formats; - if (cast_receiver_session_->format_.IsValid()) - formats.push_back(cast_receiver_session_->format_); - return formats; -} - -void CastReceiverSession::VideoCapturerSource::StartCapture( - const media::VideoCaptureParams& params, - const VideoCaptureDeliverFrameCB& frame_callback, - const RunningCallback& running_callback) { - cast_receiver_session_->StartVideo(frame_callback); - running_callback.Run(true); -} - -void CastReceiverSession::VideoCapturerSource::StopCapture() { - cast_receiver_session_->StopVideo(); -} - -CastReceiverSession::AudioCapturerSource::AudioCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session) - : cast_receiver_session_(cast_receiver_session) { -} - -CastReceiverSession::AudioCapturerSource::~AudioCapturerSource() { - DCHECK(!audio_valve_); -} - -void CastReceiverSession::AudioCapturerSource::Initialize( - const media::AudioParameters& params, - CaptureCallback* callback) { - // TODO(hubbe): Consider converting the audio to whatever the caller wants. - if (params.sample_rate() != - cast_receiver_session_->audio_config_.rtp_timebase || - params.channels() != cast_receiver_session_->audio_config_.channels) { - callback->OnCaptureError(std::string()); - return; - } - audio_valve_ = new CastReceiverAudioValve(params, callback); -} - -void CastReceiverSession::AudioCapturerSource::Start() { - DCHECK(audio_valve_); - cast_receiver_session_->StartAudio(audio_valve_); -} - -void CastReceiverSession::AudioCapturerSource::Stop() { - audio_valve_->Stop(); - audio_valve_ = nullptr; -} - -void CastReceiverSession::AudioCapturerSource::SetVolume(double volume) { - // not supported -} - -void CastReceiverSession::AudioCapturerSource::SetAutomaticGainControl( - bool enable) { - // not supported -} - -void CastReceiverSession::AudioCapturerSource::SetOutputDeviceForAec( - const std::string& output_device_id) { - // not supported -} diff --git a/chromium/chrome/renderer/media/cast_receiver_session.h b/chromium/chrome/renderer/media/cast_receiver_session.h deleted file mode 100644 index c9bfa5a1181..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_H_ -#define CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_H_ - -#include <memory> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/single_thread_task_runner.h" -#include "chrome/renderer/media/cast_receiver_session_delegate.h" - -namespace media { -class AudioCapturerSource; -struct VideoCaptureFormat; -class VideoCapturerSource; -} - -namespace net { -class IPEndPoint; -} - -namespace base { -class DictionaryValue; -} - -// This a render thread object, all methods, construction and -// destruction must happen on the render thread. -class CastReceiverSession : public base::RefCounted<CastReceiverSession> { - public: - CastReceiverSession(); - - typedef base::Callback<void(scoped_refptr<media::AudioCapturerSource>, - std::unique_ptr<media::VideoCapturerSource>)> - StartCB; - - // Note that the cast receiver will start responding to - // incoming network streams immediately, buffering input until - // StartAudio/StartVideo is called. - // Five first parameters are passed to cast receiver. - // |start_callback| is called when initialization is done. - // TODO(hubbe): Currently the audio component of the returned media - // stream only the exact format that the sender is sending us. - void Start(const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& capture_format, - const StartCB& start_callback, - const CastReceiverSessionDelegate::ErrorCallback& error_callback); - - private: - class VideoCapturerSource; - class AudioCapturerSource; - friend class base::RefCounted<CastReceiverSession>; - virtual ~CastReceiverSession(); - void StartAudio(scoped_refptr<CastReceiverAudioValve> audio_valve); - - void StartVideo(blink::VideoCaptureDeliverFrameCB frame_callback); - // Stop Video callbacks. - // Note that this returns immediately, but callbacks do not stop immediately. - void StopVideo(); - - media::cast::FrameReceiverConfig audio_config_; - media::cast::FrameReceiverConfig video_config_; - std::unique_ptr<CastReceiverSessionDelegate> delegate_; - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - media::VideoCaptureFormat format_; - - DISALLOW_COPY_AND_ASSIGN(CastReceiverSession); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_H_ diff --git a/chromium/chrome/renderer/media/cast_receiver_session_delegate.cc b/chromium/chrome/renderer/media/cast_receiver_session_delegate.cc deleted file mode 100644 index c6b6a1053b9..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session_delegate.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_receiver_session_delegate.h" - -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/synchronization/waitable_event.h" -#include "base/values.h" - -CastReceiverSessionDelegate::CastReceiverSessionDelegate() {} -CastReceiverSessionDelegate::~CastReceiverSessionDelegate() {} - -void CastReceiverSessionDelegate::Start( - const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& format, - const ErrorCallback& error_callback) { - format_ = format; - DCHECK(io_task_runner_->BelongsToCurrentThread()); - CastSessionDelegateBase::StartUDP(local_endpoint, remote_endpoint, - std::move(options), error_callback); - cast_receiver_ = media::cast::CastReceiver::Create(cast_environment_, - audio_config, - video_config, - cast_transport_.get()); - on_audio_decoded_cb_ = - base::BindRepeating(&CastReceiverSessionDelegate::OnDecodedAudioFrame, - weak_factory_.GetWeakPtr()); - on_video_decoded_cb_ = - base::BindRepeating(&CastReceiverSessionDelegate::OnDecodedVideoFrame, - weak_factory_.GetWeakPtr()); -} - -void CastReceiverSessionDelegate::ReceivePacket( - std::unique_ptr<media::cast::Packet> packet) { - cast_receiver_->ReceivePacket(std::move(packet)); -} - -void CastReceiverSessionDelegate::StartAudio( - scoped_refptr<CastReceiverAudioValve> audio_valve) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - audio_valve_ = audio_valve; - cast_receiver_->RequestDecodedAudioFrame(on_audio_decoded_cb_); -} - -void CastReceiverSessionDelegate::OnDecodedAudioFrame( - std::unique_ptr<media::AudioBus> audio_bus, - base::TimeTicks playout_time, - bool is_continuous) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - if (!audio_valve_) - return; - - // We're on the IO thread, which doesn't allow blocking - // operations. Since we don't know what the Capture callback - // will do exactly, we need to jump to a different thread. - // Let's re-use the audio decoder thread. - cast_environment_->PostTask( - media::cast::CastEnvironment::AUDIO, FROM_HERE, - base::BindOnce(&CastReceiverAudioValve::DeliverDecodedAudio, audio_valve_, - base::Owned(audio_bus.release()), playout_time)); - cast_receiver_->RequestDecodedAudioFrame(on_audio_decoded_cb_); -} - -void CastReceiverSessionDelegate::StartVideo( - blink::VideoCaptureDeliverFrameCB video_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - frame_callback_ = video_callback; - cast_receiver_->RequestDecodedVideoFrame(on_video_decoded_cb_); -} - -void CastReceiverSessionDelegate::StopVideo() { - frame_callback_ = blink::VideoCaptureDeliverFrameCB(); -} - -void CastReceiverSessionDelegate::OnDecodedVideoFrame( - scoped_refptr<media::VideoFrame> video_frame, - base::TimeTicks playout_time, - bool is_continuous) { - if (frame_callback_.is_null()) - return; - frame_callback_.Run(std::move(video_frame), playout_time); - cast_receiver_->RequestDecodedVideoFrame(on_video_decoded_cb_); -} diff --git a/chromium/chrome/renderer/media/cast_receiver_session_delegate.h b/chromium/chrome/renderer/media/cast_receiver_session_delegate.h deleted file mode 100644 index e7708cb3e2f..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session_delegate.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_DELEGATE_H_ -#define CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_DELEGATE_H_ - -#include <memory> - -#include "base/macros.h" -#include "chrome/renderer/media/cast_receiver_audio_valve.h" -#include "chrome/renderer/media/cast_session_delegate.h" -#include "media/capture/video_capture_types.h" -#include "media/cast/cast_receiver.h" -#include "third_party/blink/public/common/media/video_capture.h" - -class CastReceiverSessionDelegate : public CastSessionDelegateBase { - public: - typedef base::Callback<void(const std::string&)> ErrorCallback; - - CastReceiverSessionDelegate(); - ~CastReceiverSessionDelegate() override; - - void ReceivePacket(std::unique_ptr<media::cast::Packet> packet) override; - - void Start(const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& format, - const ErrorCallback& error_callback); - - void StartAudio(scoped_refptr<CastReceiverAudioValve> audio_valve); - - void StartVideo(blink::VideoCaptureDeliverFrameCB frame_callback); - // Stop Video callbacks (eventually). - void StopVideo(); - - private: - void OnDecodedAudioFrame(std::unique_ptr<media::AudioBus> audio_bus, - base::TimeTicks playout_time, - bool is_continuous); - - void OnDecodedVideoFrame(scoped_refptr<media::VideoFrame> video_frame, - base::TimeTicks playout_time, - bool is_continuous); - - scoped_refptr<CastReceiverAudioValve> audio_valve_; - blink::VideoCaptureDeliverFrameCB frame_callback_; - media::cast::AudioFrameDecodedCallback on_audio_decoded_cb_; - media::cast::VideoFrameDecodedCallback on_video_decoded_cb_; - std::unique_ptr<media::cast::CastReceiver> cast_receiver_; - media::VideoCaptureFormat format_; - base::WeakPtrFactory<CastReceiverSessionDelegate> weak_factory_{this}; - DISALLOW_COPY_AND_ASSIGN(CastReceiverSessionDelegate); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_DELEGATE_H_ diff --git a/chromium/chrome/renderer/media/cast_rtp_stream.cc b/chromium/chrome/renderer/media/cast_rtp_stream.cc deleted file mode 100644 index 1deb6336bbd..00000000000 --- a/chromium/chrome/renderer/media/cast_rtp_stream.cc +++ /dev/null @@ -1,578 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_rtp_stream.h" - -#include <stdint.h> - -#include <algorithm> -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/bind_helpers.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/timer/timer.h" -#include "base/trace_event/trace_event.h" -#include "base/values.h" -#include "chrome/common/chrome_switches.h" -#include "chrome/renderer/media/cast_session.h" -#include "chrome/renderer/media/cast_udp_transport.h" -#include "content/public/renderer/render_thread.h" -#include "content/public/renderer/video_encode_accelerator.h" -#include "media/base/audio_bus.h" -#include "media/base/audio_converter.h" -#include "media/base/audio_parameters.h" -#include "media/base/bind_to_current_loop.h" -#include "media/base/limits.h" -#include "media/base/video_frame.h" -#include "media/base/video_util.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_sender.h" -#include "media/cast/net/cast_transport_config.h" -#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_audio_sink.h" -#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_sink.h" -#include "third_party/blink/public/platform/web_media_stream_source.h" -#include "third_party/blink/public/web/modules/mediastream/web_media_stream_utils.h" -#include "ui/gfx/geometry/size.h" - -using media::cast::FrameSenderConfig; - -namespace { - -// The maximum number of milliseconds that should elapse since the last video -// frame was received from the video source, before requesting refresh frames. -const int kRefreshIntervalMilliseconds = 250; - -// The maximum number of refresh video frames to request/receive. After this -// limit (60 * 250ms = 15 seconds), refresh frame requests will stop being made. -const int kMaxConsecutiveRefreshFrames = 60; - -FrameSenderConfig DefaultOpusConfig() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::AUDIO_OPUS; - config.sender_ssrc = 1; - config.receiver_ssrc = 2; - config.rtp_timebase = media::cast::kDefaultAudioSamplingRate; - config.channels = 2; - config.min_bitrate = config.max_bitrate = config.start_bitrate = - media::cast::kDefaultAudioEncoderBitrate; - config.max_frame_rate = 100; // 10 ms audio frames - config.codec = media::cast::CODEC_AUDIO_OPUS; - return config; -} - -FrameSenderConfig DefaultVp8Config() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::VIDEO_VP8; - config.sender_ssrc = 11; - config.receiver_ssrc = 12; - config.rtp_timebase = media::cast::kVideoFrequency; - config.channels = 1; - config.max_bitrate = media::cast::kDefaultMaxVideoBitrate; - config.min_bitrate = media::cast::kDefaultMinVideoBitrate; - config.max_frame_rate = media::cast::kDefaultMaxFrameRate; - config.codec = media::cast::CODEC_VIDEO_VP8; - return config; -} - -FrameSenderConfig DefaultH264Config() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::VIDEO_H264; - config.sender_ssrc = 11; - config.receiver_ssrc = 12; - config.rtp_timebase = media::cast::kVideoFrequency; - config.channels = 1; - config.max_bitrate = media::cast::kDefaultMaxVideoBitrate; - config.min_bitrate = media::cast::kDefaultMinVideoBitrate; - config.max_frame_rate = media::cast::kDefaultMaxFrameRate; - config.codec = media::cast::CODEC_VIDEO_H264; - return config; -} - -FrameSenderConfig DefaultRemotingAudioConfig() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::REMOTE_AUDIO; - config.sender_ssrc = 3; - config.receiver_ssrc = 4; - config.codec = media::cast::CODEC_AUDIO_REMOTE; - config.rtp_timebase = media::cast::kRemotingRtpTimebase; - config.max_bitrate = 1000000; - config.min_bitrate = 0; - config.channels = 2; - config.max_frame_rate = 100; // 10 ms audio frames - - return config; -} - -FrameSenderConfig DefaultRemotingVideoConfig() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::REMOTE_VIDEO; - config.sender_ssrc = 13; - config.receiver_ssrc = 14; - config.codec = media::cast::CODEC_VIDEO_REMOTE; - config.rtp_timebase = media::cast::kRemotingRtpTimebase; - config.max_bitrate = 10000000; - config.min_bitrate = 0; - config.channels = 1; - config.max_frame_rate = media::cast::kDefaultMaxFrameRate; - return config; -} - -std::vector<FrameSenderConfig> SupportedAudioConfigs(bool for_remoting_stream) { - if (for_remoting_stream) - return {DefaultRemotingAudioConfig()}; - else - return {DefaultOpusConfig()}; -} - -std::vector<FrameSenderConfig> SupportedVideoConfigs(bool for_remoting_stream) { - if (for_remoting_stream) - return {DefaultRemotingVideoConfig()}; - - std::vector<FrameSenderConfig> supported_configs; - // Prefer VP8 over H.264 for hardware encoder. - if (CastRtpStream::IsHardwareVP8EncodingSupported()) - supported_configs.push_back(DefaultVp8Config()); - if (CastRtpStream::IsHardwareH264EncodingSupported()) - supported_configs.push_back(DefaultH264Config()); - - // Propose the default software VP8 encoder, if no hardware encoders are - // available. - if (supported_configs.empty()) - supported_configs.push_back(DefaultVp8Config()); - - return supported_configs; -} - -} // namespace - -// This class receives MediaStreamTrack events and video frames from a -// MediaStreamVideoTrack. It also includes a timer to request refresh frames -// when the capturer halts (e.g., a screen capturer stops delivering frames -// because the screen is not being updated). When a halt is detected, refresh -// frames will be requested at regular intervals for a short period of time. -// This provides the video encoder, downstream, several copies of the last frame -// so that it may clear up lossy encoding artifacts. -// -// Threading: Video frames are received on the IO thread and then -// forwarded to media::cast::VideoFrameInput. The inner class, Deliverer, -// handles this. Otherwise, all methods and member variables of the outer class -// must only be accessed on the render thread. -class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>, - public blink::WebMediaStreamSink { - public: - // |track| provides data for this sink. - // |error_callback| is called if video formats don't match. - CastVideoSink(const blink::WebMediaStreamTrack& track, - CastRtpStream::ErrorCallback error_callback) - : track_(track), - deliverer_(base::MakeRefCounted<Deliverer>(std::move(error_callback))), - consecutive_refresh_count_(0), - expecting_a_refresh_frame_(false), - is_connected_to_track_(false) {} - - ~CastVideoSink() override { - if (is_connected_to_track_) - blink::RemoveSinkFromMediaStreamTrack(track_, this); - } - - // Attach this sink to a video track represented by |track_|. - // Data received from the track will be submitted to |frame_input|. - void AddToTrack( - bool is_sink_secure, - const scoped_refptr<media::cast::VideoFrameInput>& frame_input) { - DCHECK(deliverer_); - deliverer_->WillConnectToTrack(AsWeakPtr(), frame_input); - refresh_timer_.Start( - FROM_HERE, - base::TimeDelta::FromMilliseconds(kRefreshIntervalMilliseconds), - base::Bind(&CastVideoSink::OnRefreshTimerFired, - base::Unretained(this))); - blink::AddSinkToMediaStreamTrack( - track_, this, base::BindRepeating(&Deliverer::OnVideoFrame, deliverer_), - is_sink_secure); - is_connected_to_track_ = true; - } - - private: - class Deliverer : public base::RefCountedThreadSafe<Deliverer> { - public: - explicit Deliverer(CastRtpStream::ErrorCallback error_callback) - : main_task_runner_(base::ThreadTaskRunnerHandle::Get()), - error_callback_(std::move(error_callback)) {} - - void WillConnectToTrack( - base::WeakPtr<CastVideoSink> sink, - scoped_refptr<media::cast::VideoFrameInput> frame_input) { - DCHECK(main_task_runner_->RunsTasksInCurrentSequence()); - sink_ = sink; - frame_input_ = std::move(frame_input); - } - - void OnVideoFrame(scoped_refptr<media::VideoFrame> video_frame, - base::TimeTicks estimated_capture_time) { - main_task_runner_->PostTask( - FROM_HERE, base::BindOnce(&CastVideoSink::DidReceiveFrame, sink_)); - - const base::TimeTicks timestamp = estimated_capture_time.is_null() - ? base::TimeTicks::Now() - : estimated_capture_time; - - if (!(video_frame->format() == media::PIXEL_FORMAT_I420 || - video_frame->format() == media::PIXEL_FORMAT_YV12 || - video_frame->format() == media::PIXEL_FORMAT_I420A)) { - error_callback_.Run("Incompatible video frame format."); - return; - } - scoped_refptr<media::VideoFrame> frame = video_frame; - // Drop alpha channel since we do not support it yet. - if (frame->format() == media::PIXEL_FORMAT_I420A) - frame = media::WrapAsI420VideoFrame(std::move(video_frame)); - - // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc - TRACE_EVENT_INSTANT2("cast_perf_test", "ConsumeVideoFrame", - TRACE_EVENT_SCOPE_THREAD, "timestamp", - (timestamp - base::TimeTicks()).InMicroseconds(), - "time_delta", frame->timestamp().InMicroseconds()); - frame_input_->InsertRawVideoFrame(std::move(frame), timestamp); - } - - private: - friend class base::RefCountedThreadSafe<Deliverer>; - ~Deliverer() {} - - const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; - const CastRtpStream::ErrorCallback error_callback_; - - // These are set on the main thread after construction, and before the first - // call to OnVideoFrame() on the IO thread. |sink_| may be passed around on - // any thread, but must only be dereferenced on the main renderer thread. - base::WeakPtr<CastVideoSink> sink_; - scoped_refptr<media::cast::VideoFrameInput> frame_input_; - - DISALLOW_COPY_AND_ASSIGN(Deliverer); - }; - - private: - void OnRefreshTimerFired() { - ++consecutive_refresh_count_; - if (consecutive_refresh_count_ >= kMaxConsecutiveRefreshFrames) - refresh_timer_.Stop(); // Stop timer until receiving a non-refresh frame. - - DVLOG(1) << "CastVideoSink is requesting another refresh frame " - "(consecutive count=" << consecutive_refresh_count_ << ")."; - expecting_a_refresh_frame_ = true; - blink::RequestRefreshFrameFromVideoTrack(track_); - } - - void DidReceiveFrame() { - if (expecting_a_refresh_frame_) { - // There is uncertainty as to whether the video frame was in response to a - // refresh request. However, if it was not, more video frames will soon - // follow, and before the refresh timer can fire again. Thus, the - // behavior resulting from this logic will be correct. - expecting_a_refresh_frame_ = false; - } else { - consecutive_refresh_count_ = 0; - // The following re-starts the timer, scheduling it to fire at - // kRefreshIntervalMilliseconds from now. - refresh_timer_.Reset(); - } - } - - const blink::WebMediaStreamTrack track_; - const scoped_refptr<Deliverer> deliverer_; - - // Requests refresh frames at a constant rate while the source is paused, up - // to a consecutive maximum. - base::RepeatingTimer refresh_timer_; - - // Counter for the number of consecutive "refresh frames" requested. - int consecutive_refresh_count_; - - // Set to true when a request for a refresh frame has been made. This is - // cleared once the next frame is received. - bool expecting_a_refresh_frame_; - - bool is_connected_to_track_; - - DISALLOW_COPY_AND_ASSIGN(CastVideoSink); -}; - -// Receives audio data from a MediaStreamTrack. Data is submitted to -// media::cast::FrameInput. -// -// Threading: Audio frames are received on the real-time audio thread. -// Note that RemoveFromAudioTrack() is synchronous and we have -// gurantee that there will be no more audio data after calling it. -class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>, - public blink::WebMediaStreamAudioSink, - public media::AudioConverter::InputCallback { - public: - // |track| provides data for this sink. - CastAudioSink(const blink::WebMediaStreamTrack& track, - int output_channels, - int output_sample_rate) - : track_(track), - output_channels_(output_channels), - output_sample_rate_(output_sample_rate), - current_input_bus_(nullptr), - sample_frames_in_(0), - sample_frames_out_(0) {} - - ~CastAudioSink() override { - if (frame_input_.get()) - RemoveFromAudioTrack(this, track_); - } - - // Add this sink to the track. Data received from the track will be - // submitted to |frame_input|. - void AddToTrack( - const scoped_refptr<media::cast::AudioFrameInput>& frame_input) { - DCHECK(frame_input.get()); - DCHECK(!frame_input_.get()); - // This member is written here and then accessed on the IO thread - // We will not get data until AddToAudioTrack is called so it is - // safe to access this member now. - frame_input_ = frame_input; - AddToAudioTrack(this, track_); - } - - protected: - // Called on real-time audio thread. - void OnData(const media::AudioBus& input_bus, - base::TimeTicks estimated_capture_time) override { - DCHECK(input_params_.IsValid()); - DCHECK_EQ(input_bus.channels(), input_params_.channels()); - DCHECK_EQ(input_bus.frames(), input_params_.frames_per_buffer()); - DCHECK(!estimated_capture_time.is_null()); - DCHECK(converter_.get()); - - // Determine the duration of the audio signal enqueued within |converter_|. - const base::TimeDelta signal_duration_already_buffered = - (sample_frames_in_ * base::TimeDelta::FromSeconds(1) / - input_params_.sample_rate()) - - (sample_frames_out_ * base::TimeDelta::FromSeconds(1) / - output_sample_rate_); - DVLOG(2) << "Audio reference time adjustment: -(" - << signal_duration_already_buffered.InMicroseconds() << " us)"; - const base::TimeTicks capture_time_of_first_converted_sample = - estimated_capture_time - signal_duration_already_buffered; - - // Convert the entire input signal. AudioConverter is efficient in that no - // additional copying or conversion will occur if the input signal is in the - // same format as the output. Note that, while the number of sample frames - // provided as input is always the same, the chunk size (and the size of the - // |audio_bus| here) can be variable. This is not an issue since - // media::cast::AudioFrameInput can handle variable-sized AudioBuses. - std::unique_ptr<media::AudioBus> audio_bus = - media::AudioBus::Create(output_channels_, converter_->ChunkSize()); - // AudioConverter will call ProvideInput() to fetch from |current_data_|. - current_input_bus_ = &input_bus; - converter_->Convert(audio_bus.get()); - DCHECK(!current_input_bus_); // ProvideInput() called exactly once? - - sample_frames_in_ += input_params_.frames_per_buffer(); - sample_frames_out_ += audio_bus->frames(); - - frame_input_->InsertAudio(std::move(audio_bus), - capture_time_of_first_converted_sample); - } - - // Called on real-time audio thread. - void OnSetFormat(const media::AudioParameters& params) override { - if (input_params_.Equals(params)) - return; - input_params_ = params; - - DVLOG(1) << "Setting up audio resampling: {" - << input_params_.channels() << " channels, " - << input_params_.sample_rate() << " Hz} --> {" - << output_channels_ << " channels, " - << output_sample_rate_ << " Hz}"; - const media::AudioParameters output_params( - media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::GuessChannelLayout(output_channels_), output_sample_rate_, - output_sample_rate_ * input_params_.frames_per_buffer() / - input_params_.sample_rate()); - converter_.reset( - new media::AudioConverter(input_params_, output_params, false)); - converter_->AddInput(this); - sample_frames_in_ = 0; - sample_frames_out_ = 0; - } - - // Called on real-time audio thread. - double ProvideInput(media::AudioBus* audio_bus, - uint32_t frames_delayed) override { - DCHECK(current_input_bus_); - current_input_bus_->CopyTo(audio_bus); - current_input_bus_ = nullptr; - return 1.0; - } - - private: - const blink::WebMediaStreamTrack track_; - const int output_channels_; - const int output_sample_rate_; - - // This must be set before the real-time audio thread starts calling OnData(), - // and remain unchanged until after the thread will stop calling OnData(). - scoped_refptr<media::cast::AudioFrameInput> frame_input_; - - // These members are accessed on the real-time audio time only. - media::AudioParameters input_params_; - std::unique_ptr<media::AudioConverter> converter_; - const media::AudioBus* current_input_bus_; - int64_t sample_frames_in_; - int64_t sample_frames_out_; - - DISALLOW_COPY_AND_ASSIGN(CastAudioSink); -}; - -bool CastRtpStream::IsHardwareVP8EncodingSupported() { - // Query for hardware VP8 encoder support. - const std::vector<media::VideoEncodeAccelerator::SupportedProfile> - vea_profiles = content::GetSupportedVideoEncodeAcceleratorProfiles(); - for (const auto& vea_profile : vea_profiles) { - if (vea_profile.profile >= media::VP8PROFILE_MIN && - vea_profile.profile <= media::VP8PROFILE_MAX) { - return true; - } - } - return false; -} - -bool CastRtpStream::IsHardwareH264EncodingSupported() { -// Query for hardware H.264 encoder support. -// -// TODO(miu): Look into why H.264 hardware encoder on MacOS is broken. -// http://crbug.com/596674 -// TODO(emircan): Look into HW encoder initialization issues on Win. -// https://crbug.com/636064 -#if !defined(OS_MACOSX) && !defined(OS_WIN) - const std::vector<media::VideoEncodeAccelerator::SupportedProfile> - vea_profiles = content::GetSupportedVideoEncodeAcceleratorProfiles(); - for (const auto& vea_profile : vea_profiles) { - if (vea_profile.profile >= media::H264PROFILE_MIN && - vea_profile.profile <= media::H264PROFILE_MAX) { - return true; - } - } -#endif // !defined(OS_MACOSX) && !defined(OS_WIN) - return false; -} - -CastRtpStream::CastRtpStream(const blink::WebMediaStreamTrack& track, - const scoped_refptr<CastSession>& session) - : track_(track), - cast_session_(session), - is_audio_(track_.Source().GetType() == - blink::WebMediaStreamSource::kTypeAudio) {} - -CastRtpStream::CastRtpStream(bool is_audio, - const scoped_refptr<CastSession>& session) - : cast_session_(session), is_audio_(is_audio) {} - -CastRtpStream::~CastRtpStream() { - Stop(); -} - -std::vector<FrameSenderConfig> CastRtpStream::GetSupportedConfigs() { - if (is_audio_) - return SupportedAudioConfigs(track_.IsNull()); - else - return SupportedVideoConfigs(track_.IsNull()); -} - -void CastRtpStream::Start(int32_t stream_id, - const FrameSenderConfig& config, - base::OnceClosure start_callback, - base::OnceClosure stop_callback, - ErrorCallback error_callback) { - DCHECK(!start_callback.is_null()); - DCHECK(!stop_callback.is_null()); - DCHECK(!error_callback.is_null()); - - DVLOG(1) << "CastRtpStream::Start = " << (is_audio_ ? "audio" : "video"); - stop_callback_ = std::move(stop_callback); - error_callback_ = std::move(error_callback); - - if (track_.IsNull()) { - cast_session_->StartRemotingStream( - stream_id, config, - base::BindOnce(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())); - } else if (is_audio_) { - // In case of error we have to go through DidEncounterError() to stop - // the streaming after reporting the error. - audio_sink_.reset( - new CastAudioSink(track_, config.channels, config.rtp_timebase)); - cast_session_->StartAudio( - config, - base::Bind(&CastAudioSink::AddToTrack, audio_sink_->AsWeakPtr()), - base::Bind(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())); - } else { - // See the code for audio above for explanation of callbacks. - video_sink_.reset(new CastVideoSink( - track_, - media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())))); - cast_session_->StartVideo( - config, base::Bind(&CastVideoSink::AddToTrack, video_sink_->AsWeakPtr(), - !config.aes_key.empty()), - base::Bind(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())); - } - std::move(start_callback).Run(); -} - -void CastRtpStream::Stop() { - DVLOG(1) << "CastRtpStream::Stop = " << (is_audio_ ? "audio" : "video"); - if (stop_callback_.is_null()) - return; // Already stopped. - weak_factory_.InvalidateWeakPtrs(); - error_callback_.Reset(); - audio_sink_.reset(); - video_sink_.reset(); - std::move(stop_callback_).Run(); -} - -void CastRtpStream::ToggleLogging(bool enable) { - DVLOG(1) << "CastRtpStream::ToggleLogging(" << enable - << ") = " << (is_audio_ ? "audio" : "video"); - cast_session_->ToggleLogging(is_audio_, enable); -} - -void CastRtpStream::GetRawEvents( - const base::Callback<void(std::unique_ptr<base::Value>)>& callback, - const std::string& extra_data) { - DVLOG(1) << "CastRtpStream::GetRawEvents = " - << (is_audio_ ? "audio" : "video"); - cast_session_->GetEventLogsAndReset(is_audio_, extra_data, callback); -} - -void CastRtpStream::GetStats( - const base::Callback<void(std::unique_ptr<base::DictionaryValue>)>& - callback) { - DVLOG(1) << "CastRtpStream::GetStats = " << (is_audio_ ? "audio" : "video"); - cast_session_->GetStatsAndReset(is_audio_, callback); -} - -void CastRtpStream::DidEncounterError(const std::string& message) { - DCHECK(content::RenderThread::Get()); - DVLOG(1) << "CastRtpStream::DidEncounterError(" << message - << ") = " << (is_audio_ ? "audio" : "video"); - // Save the WeakPtr first because the error callback might delete this object. - base::WeakPtr<CastRtpStream> ptr = weak_factory_.GetWeakPtr(); - error_callback_.Run(message); - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(&CastRtpStream::Stop, ptr)); -} diff --git a/chromium/chrome/renderer/media/cast_rtp_stream.h b/chromium/chrome/renderer/media/cast_rtp_stream.h deleted file mode 100644 index f39eb55f724..00000000000 --- a/chromium/chrome/renderer/media/cast_rtp_stream.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RTP_STREAM_H_ -#define CHROME_RENDERER_MEDIA_CAST_RTP_STREAM_H_ - -#include <memory> -#include <string> -#include <vector> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "media/cast/cast_config.h" -#include "media/cast/constants.h" -#include "third_party/blink/public/platform/web_media_stream_track.h" - -namespace base { -class DictionaryValue; -class Value; -} - -class CastAudioSink; -class CastSession; -class CastVideoSink; - -// This object represents a RTP stream that encodes and optionally -// encrypt audio or video data from a WebMediaStreamTrack. -// Note that this object does not actually output packets. It allows -// configuration of encoding and RTP parameters and control such a logical -// stream. -class CastRtpStream { - public: - using ErrorCallback = base::RepeatingCallback<void(const std::string&)>; - - static bool IsHardwareVP8EncodingSupported(); - - static bool IsHardwareH264EncodingSupported(); - - CastRtpStream(const blink::WebMediaStreamTrack& track, - const scoped_refptr<CastSession>& session); - CastRtpStream(bool is_audio, const scoped_refptr<CastSession>& session); - ~CastRtpStream(); - - // Return parameters currently supported by this stream. - std::vector<media::cast::FrameSenderConfig> GetSupportedConfigs(); - - // Begin encoding of media stream and then submit the encoded streams - // to underlying transport. - // |stream_id| is the unique ID of this stream. - // When the stream is started |start_callback| is called. - // When the stream is stopped |stop_callback| is called. - // When there is an error |error_callback| is called with a message. - void Start(int32_t stream_id, - const media::cast::FrameSenderConfig& config, - base::OnceClosure start_callback, - base::OnceClosure stop_callback, - ErrorCallback error_callback); - - // Stop encoding. - void Stop(); - - // Enables or disables logging for this stream. - void ToggleLogging(bool enable); - - // Get serialized raw events for this stream with |extra_data| attached, - // and invokes |callback| with the result. - void GetRawEvents( - const base::Callback<void(std::unique_ptr<base::Value>)>& callback, - const std::string& extra_data); - - // Get stats in DictionaryValue format and invokves |callback| with - // the result. - void GetStats(const base::Callback< - void(std::unique_ptr<base::DictionaryValue>)>& callback); - - private: - void DidEncounterError(const std::string& message); - - blink::WebMediaStreamTrack track_; - const scoped_refptr<CastSession> cast_session_; - std::unique_ptr<CastAudioSink> audio_sink_; - std::unique_ptr<CastVideoSink> video_sink_; - base::OnceClosure stop_callback_; - ErrorCallback error_callback_; - bool is_audio_; - - base::WeakPtrFactory<CastRtpStream> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastRtpStream); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RTP_STREAM_H_ diff --git a/chromium/chrome/renderer/media/cast_session.cc b/chromium/chrome/renderer/media/cast_session.cc deleted file mode 100644 index fd031a658a3..00000000000 --- a/chromium/chrome/renderer/media/cast_session.cc +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_session.h" - -#include <stddef.h> - -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/memory/unsafe_shared_memory_region.h" -#include "base/single_thread_task_runner.h" -#include "chrome/renderer/media/cast_session_delegate.h" -#include "content/public/renderer/render_thread.h" -#include "content/public/renderer/video_encode_accelerator.h" -#include "media/base/bind_to_current_loop.h" -#include "media/base/video_frame.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/logging_defines.h" - -namespace { - -void CreateVideoEncodeAccelerator( - const media::cast::ReceiveVideoEncodeAcceleratorCallback& callback) { - DCHECK(content::RenderThread::Get()); - - // Delegate the call to content API on the render thread. - content::CreateVideoEncodeAccelerator(callback); -} - -void CreateVideoEncodeMemory( - size_t size, - const media::cast::ReceiveVideoEncodeMemoryCallback& callback) { - DCHECK(content::RenderThread::Get()); - - base::UnsafeSharedMemoryRegion shm = - base::UnsafeSharedMemoryRegion::Create(size); - DCHECK(shm.IsValid()) << "Failed to allocate shared memory"; - callback.Run(std::move(shm)); -} - -} // namespace - -CastSession::CastSession( - scoped_refptr<base::SingleThreadTaskRunner> task_runner) - : delegate_(new CastSessionDelegate()), - main_thread_task_runner_(std::move(task_runner)), - io_task_runner_(content::RenderThread::Get()->GetIOTaskRunner()) {} - -CastSession::~CastSession() { - // We should always be able to delete the object on the IO thread. - CHECK(io_task_runner_->DeleteSoon(FROM_HERE, delegate_.release())); -} - -void CastSession::StartAudio(const media::cast::FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback) { - DCHECK(content::RenderThread::Get()); - - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce( - &CastSessionDelegate::StartAudio, base::Unretained(delegate_.get()), - config, media::BindToLoop(main_thread_task_runner_, callback), - media::BindToLoop(main_thread_task_runner_, error_callback))); -} - -void CastSession::StartVideo(const media::cast::FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback) { - DCHECK(content::RenderThread::Get()); - - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce( - &CastSessionDelegate::StartVideo, base::Unretained(delegate_.get()), - config, media::BindToLoop(main_thread_task_runner_, callback), - media::BindToLoop(main_thread_task_runner_, error_callback), - media::BindToLoop(main_thread_task_runner_, - base::Bind(&CreateVideoEncodeAccelerator)), - media::BindToLoop(main_thread_task_runner_, - base::Bind(&CreateVideoEncodeMemory)))); -} - -void CastSession::StartRemotingStream( - int32_t stream_id, - const media::cast::FrameSenderConfig& config, - ErrorOnceCallback error_callback) { - DCHECK(content::RenderThread::Get()); - - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::StartRemotingStream, - base::Unretained(delegate_.get()), stream_id, config, - media::BindToLoop(main_thread_task_runner_, - std::move(error_callback)))); -} - -void CastSession::StartUDP(const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce( - &CastSessionDelegate::StartUDP, base::Unretained(delegate_.get()), - net::IPEndPoint(), remote_endpoint, std::move(options), - media::BindToLoop(main_thread_task_runner_, error_callback))); -} - -void CastSession::ToggleLogging(bool is_audio, bool enable) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::ToggleLogging, - base::Unretained(delegate_.get()), is_audio, enable)); -} - -void CastSession::GetEventLogsAndReset( - bool is_audio, const std::string& extra_data, - const EventLogsCallback& callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::GetEventLogsAndReset, - base::Unretained(delegate_.get()), is_audio, extra_data, - media::BindToLoop(main_thread_task_runner_, callback))); -} - -void CastSession::GetStatsAndReset(bool is_audio, - const StatsCallback& callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::GetStatsAndReset, - base::Unretained(delegate_.get()), is_audio, - media::BindToLoop(main_thread_task_runner_, callback))); -} diff --git a/chromium/chrome/renderer/media/cast_session.h b/chromium/chrome/renderer/media/cast_session.h deleted file mode 100644 index 6a3f79e04fd..00000000000 --- a/chromium/chrome/renderer/media/cast_session.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_SESSION_H_ -#define CHROME_RENDERER_MEDIA_CAST_SESSION_H_ - -#include <memory> -#include <vector> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "media/cast/cast_config.h" -#include "net/base/ip_endpoint.h" - -namespace base { -class DictionaryValue; -class SingleThreadTaskRunner; -class Value; -} // namespace base - -namespace media { -namespace cast { -class AudioFrameInput; -class VideoFrameInput; -} // namespace cast -} // namespace media - -class CastSessionDelegate; - -// This class represents a Cast session and allows the session to be -// configured on the main thread. Actual work is forwarded to -// CastSessionDelegate on the IO thread. -class CastSession : public base::RefCounted<CastSession> { - public: - using AudioFrameInputAvailableCallback = - base::Callback<void(const scoped_refptr<media::cast::AudioFrameInput>&)>; - using VideoFrameInputAvailableCallback = - base::Callback<void(const scoped_refptr<media::cast::VideoFrameInput>&)>; - using SendPacketCallback = base::Callback<void(const std::vector<char>&)>; - using EventLogsCallback = base::Callback<void(std::unique_ptr<base::Value>)>; - using StatsCallback = - base::Callback<void(std::unique_ptr<base::DictionaryValue>)>; - // TODO(crbug.com/1007641): remove ErrorCallback and rename ErrorOnceCallback - // once all occurrences of base::Callback have been removed. - using ErrorCallback = base::Callback<void(const std::string&)>; - using ErrorOnceCallback = base::OnceCallback<void(const std::string&)>; - - explicit CastSession(scoped_refptr<base::SingleThreadTaskRunner> task_runner); - - // Start encoding of audio and video using the provided configuration. - // - // When Cast sender is started and ready to be used - // media::cast::FrameInput will be given through |callback|. - // If it encounters an error, |error_callback| will be invoked with the - // error message. Both |callback| and |error_callback| will be made on - // the main thread. - // |StartUDP()| must be called before these methods. - void StartAudio(const media::cast::FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback); - void StartVideo(const media::cast::FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback); - - // Start remoting a stream. |error_callback| will be invoked when any error - // occurs. |StartUDP()| must be called before calling this method. - void StartRemotingStream(int32_t stream_id, - const media::cast::FrameSenderConfig& config, - ErrorOnceCallback error_callback); - - // This will create the Cast transport and connect to |remote_endpoint|. - // |options| is a dictionary which contain optional configuration for the - // udp transport. - // Must be called before initialization of audio or video. - void StartUDP(const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback); - - // Creates or destroys event subscriber for the audio or video stream. - // |is_audio|: true if the event subscriber is for audio. Video otherwise. - // |enable|: If true, creates an event subscriber. Otherwise destroys - // existing subscriber and discards logs. - void ToggleLogging(bool is_audio, bool enable); - - // Returns raw event logs in serialized format for either the audio or video - // stream since last call and returns result in |callback|. Also attaches - // |extra_data| to the log. - void GetEventLogsAndReset(bool is_audio, - const std::string& extra_data, const EventLogsCallback& callback); - - // Returns stats in a DictionaryValue format for either the audio or video - // stream since last call and returns result in |callback|. - void GetStatsAndReset(bool is_audio, const StatsCallback& callback); - - private: - friend class base::RefCounted<CastSession>; - virtual ~CastSession(); - - // This member should never be dereferenced on the main thread. - // CastSessionDelegate lives only on the IO thread. It is always - // safe to post task on the IO thread to access CastSessionDelegate - // because it is owned by this object. - std::unique_ptr<CastSessionDelegate> delegate_; - - // A main thread task runner that might execute JavaScript. - const scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner_; - - // Proxy to the IO task runner. - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - - DISALLOW_COPY_AND_ASSIGN(CastSession); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_SESSION_H_ diff --git a/chromium/chrome/renderer/media/cast_session_browsertest.cc b/chromium/chrome/renderer/media/cast_session_browsertest.cc deleted file mode 100644 index c4c4337d7e8..00000000000 --- a/chromium/chrome/renderer/media/cast_session_browsertest.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_session.h" - -#include "base/run_loop.h" -#include "base/threading/thread_task_runner_handle.h" -#include "chrome/renderer/chrome_content_renderer_client.h" -#include "chrome/test/base/chrome_render_view_test.h" -#include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h" - -typedef ChromeRenderViewTest CastSessionBrowserTest; - -// Tests that CastSession is created and destroyed properly inside -// chrome renderer. -TEST_F(CastSessionBrowserTest, CreateAndDestroy) { - chrome_render_thread_->set_io_task_runner( - blink::scheduler::GetSingleThreadTaskRunnerForTesting()); - ChromeContentRendererClient* client = - static_cast<ChromeContentRendererClient*>(content_renderer_client_.get()); - client->RenderThreadStarted(); - - scoped_refptr<CastSession> session( - new CastSession(blink::scheduler::GetSingleThreadTaskRunnerForTesting())); - - // Causes CastSession to destruct. - session.reset(); - base::RunLoop().RunUntilIdle(); -} diff --git a/chromium/chrome/renderer/media/cast_session_delegate.cc b/chromium/chrome/renderer/media/cast_session_delegate.cc deleted file mode 100644 index 7bc22f2ecc5..00000000000 --- a/chromium/chrome/renderer/media/cast_session_delegate.cc +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_session_delegate.h" - -#include <memory> -#include <utility> -#include <vector> - -#include "base/bind.h" -#include "base/lazy_instance.h" -#include "base/logging.h" -#include "base/single_thread_task_runner.h" -#include "base/strings/stringprintf.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/values.h" -#include "build/build_config.h" -#include "chrome/renderer/media/cast_threads.h" -#include "chrome/renderer/media/cast_transport_ipc.h" -#include "components/version_info/version_info.h" -#include "content/public/renderer/render_thread.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_environment.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/log_serializer.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/logging/proto/raw_events.pb.h" -#include "media/cast/logging/raw_event_subscriber_bundle.h" -#include "media/cast/net/cast_transport.h" -#include "media/cast/net/cast_transport_config.h" - -using media::cast::CastEnvironment; -using media::cast::CastSender; -using media::cast::FrameSenderConfig; - -static base::LazyInstance<CastThreads>::DestructorAtExit g_cast_threads = - LAZY_INSTANCE_INITIALIZER; - -CastSessionDelegateBase::CastSessionDelegateBase() - : io_task_runner_(content::RenderThread::Get()->GetIOTaskRunner()) { - DCHECK(io_task_runner_.get()); -#if defined(OS_WIN) - // Note that this also increases the accuracy of PostDelayTask, - // which is is very helpful to cast. - if (!base::Time::ActivateHighResolutionTimer(true)) { - LOG(WARNING) << "Failed to activate high resolution timers for cast."; - } -#endif -} - -CastSessionDelegateBase::~CastSessionDelegateBase() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); -#if defined(OS_WIN) - base::Time::ActivateHighResolutionTimer(false); -#endif -} - -void CastSessionDelegateBase::StartUDP( - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - // CastSender uses the renderer's IO thread as the main thread. This reduces - // thread hopping for incoming video frames and outgoing network packets. - // TODO(hubbe): Create cast environment in ctor instead. - cast_environment_ = - new CastEnvironment(base::DefaultTickClock::GetInstance(), - base::ThreadTaskRunnerHandle::Get(), - g_cast_threads.Get().GetAudioEncodeTaskRunner(), - g_cast_threads.Get().GetVideoEncodeTaskRunner()); - - // Rationale for using unretained: The callback cannot be called after the - // destruction of CastTransportIPC, and they both share the same thread. - cast_transport_ = std::make_unique<CastTransportIPC>( - local_endpoint, remote_endpoint, std::move(options), - base::BindRepeating(&CastSessionDelegateBase::ReceivePacket, - base::Unretained(this)), - base::BindRepeating(&CastSessionDelegateBase::StatusNotificationCB, - base::Unretained(this), error_callback), - base::BindRepeating( - &media::cast::LogEventDispatcher::DispatchBatchOfEvents, - base::Unretained(cast_environment_->logger()))); -} - -void CastSessionDelegateBase::StatusNotificationCB( - ErrorOnceCallback error_callback, - media::cast::CastTransportStatus status) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - std::string error_message; - - switch (status) { - case media::cast::TRANSPORT_STREAM_UNINITIALIZED: - case media::cast::TRANSPORT_STREAM_INITIALIZED: - return; // Not errors, do nothing. - case media::cast::TRANSPORT_INVALID_CRYPTO_CONFIG: - std::move(error_callback).Run("Invalid encrypt/decrypt configuration."); - break; - case media::cast::TRANSPORT_SOCKET_ERROR: - std::move(error_callback).Run("Socket error."); - break; - } -} - -CastSessionDelegate::CastSessionDelegate() { - DCHECK(io_task_runner_.get()); -} - -CastSessionDelegate::~CastSessionDelegate() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); -} - -void CastSessionDelegate::StartAudio( - const FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - ErrorOnceCallback error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!cast_transport_ || !cast_sender_) { - std::move(error_callback).Run("Destination not set."); - return; - } - - audio_frame_input_available_callback_ = callback; - cast_sender_->InitializeAudio( - config, base::BindOnce(&CastSessionDelegate::OnOperationalStatusChange, - weak_factory_.GetWeakPtr(), true, - std::move(error_callback))); -} - -void CastSessionDelegate::StartVideo( - const FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback, - const media::cast::CreateVideoEncodeAcceleratorCallback& create_vea_cb, - const media::cast::CreateVideoEncodeMemoryCallback& - create_video_encode_mem_cb) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!cast_transport_ || !cast_sender_) { - error_callback.Run("Destination not set."); - return; - } - - video_frame_input_available_callback_ = callback; - - cast_sender_->InitializeVideo( - config, - base::BindRepeating(&CastSessionDelegate::OnOperationalStatusChange, - weak_factory_.GetWeakPtr(), false, error_callback), - create_vea_cb, create_video_encode_mem_cb); -} - -void CastSessionDelegate::StartRemotingStream( - int32_t stream_id, - const FrameSenderConfig& config, - ErrorOnceCallback error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!cast_transport_) { - std::move(error_callback).Run("Destination not set."); - return; - } - - media::cast::CastTransportRtpConfig transport_config; - transport_config.ssrc = config.sender_ssrc; - transport_config.feedback_ssrc = config.receiver_ssrc; - transport_config.rtp_payload_type = config.rtp_payload_type; - transport_config.rtp_stream_id = stream_id; - transport_config.aes_key = config.aes_key; - transport_config.aes_iv_mask = config.aes_iv_mask; - cast_transport_->InitializeStream(transport_config, nullptr); -} - -void CastSessionDelegate::StartUDP( - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - CastSessionDelegateBase::StartUDP(local_endpoint, remote_endpoint, - std::move(options), error_callback); - event_subscribers_ = std::make_unique<media::cast::RawEventSubscriberBundle>( - cast_environment_); - - cast_sender_ = CastSender::Create(cast_environment_, cast_transport_.get()); -} - -void CastSessionDelegate::ToggleLogging(bool is_audio, bool enable) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - if (!event_subscribers_.get()) - return; - - if (enable) - event_subscribers_->AddEventSubscribers(is_audio); - else - event_subscribers_->RemoveEventSubscribers(is_audio); -} - -void CastSessionDelegate::GetEventLogsAndReset( - bool is_audio, - const std::string& extra_data, - const EventLogsCallback& callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!event_subscribers_.get()) { - callback.Run(std::make_unique<base::Value>(base::Value::Type::BINARY)); - return; - } - - media::cast::EncodingEventSubscriber* subscriber = - event_subscribers_->GetEncodingEventSubscriber(is_audio); - if (!subscriber) { - callback.Run(std::make_unique<base::Value>(base::Value::Type::BINARY)); - return; - } - - media::cast::proto::LogMetadata metadata; - media::cast::FrameEventList frame_events; - media::cast::PacketEventList packet_events; - - subscriber->GetEventsAndReset(&metadata, &frame_events, &packet_events); - - if (!extra_data.empty()) - metadata.set_extra_data(extra_data); - media::cast::proto::GeneralDescription* gen_desc = - metadata.mutable_general_description(); - gen_desc->set_product(version_info::GetProductName()); - gen_desc->set_product_version(version_info::GetVersionNumber()); - gen_desc->set_os(version_info::GetOSType()); - - std::unique_ptr<char[]> serialized_log( - new char[media::cast::kMaxSerializedBytes]); - int output_bytes; - bool success = media::cast::SerializeEvents(metadata, - frame_events, - packet_events, - true, - media::cast::kMaxSerializedBytes, - serialized_log.get(), - &output_bytes); - - if (!success) { - DVLOG(2) << "Failed to serialize event log."; - callback.Run(std::make_unique<base::Value>(base::Value::Type::BINARY)); - return; - } - - DVLOG(2) << "Serialized log length: " << output_bytes; - - auto blob = std::make_unique<base::Value>(std::vector<char>( - serialized_log.get(), serialized_log.get() + output_bytes)); - callback.Run(std::move(blob)); -} - -void CastSessionDelegate::GetStatsAndReset(bool is_audio, - const StatsCallback& callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!event_subscribers_.get()) { - callback.Run(std::make_unique<base::DictionaryValue>()); - return; - } - - media::cast::StatsEventSubscriber* subscriber = - event_subscribers_->GetStatsEventSubscriber(is_audio); - if (!subscriber) { - callback.Run(std::make_unique<base::DictionaryValue>()); - return; - } - - std::unique_ptr<base::DictionaryValue> stats = subscriber->GetStats(); - subscriber->Reset(); - - callback.Run(std::move(stats)); -} - -void CastSessionDelegate::OnOperationalStatusChange( - bool is_for_audio, - ErrorOnceCallback error_callback, - media::cast::OperationalStatus status) { - DCHECK(cast_sender_); - - switch (status) { - case media::cast::STATUS_UNINITIALIZED: - case media::cast::STATUS_CODEC_REINIT_PENDING: - // Not an error. - // TODO(miu): As an optimization, signal the client to pause sending more - // frames until the state becomes STATUS_INITIALIZED again. - break; - case media::cast::STATUS_INITIALIZED: - // Once initialized, run the "frame input available" callback to allow the - // client to begin sending frames. If STATUS_INITIALIZED is encountered - // again, do nothing since this is only an indication that the codec has - // successfully re-initialized. - if (is_for_audio) { - if (!audio_frame_input_available_callback_.is_null()) { - std::move(audio_frame_input_available_callback_) - .Run(cast_sender_->audio_frame_input()); - } - } else { - if (!video_frame_input_available_callback_.is_null()) { - std::move(video_frame_input_available_callback_) - .Run(cast_sender_->video_frame_input()); - } - } - break; - case media::cast::STATUS_INVALID_CONFIGURATION: - std::move(error_callback) - .Run(base::StringPrintf("Invalid %s configuration.", - is_for_audio ? "audio" : "video")); - break; - case media::cast::STATUS_UNSUPPORTED_CODEC: - std::move(error_callback) - .Run(base::StringPrintf("%s codec not supported.", - is_for_audio ? "Audio" : "Video")); - break; - case media::cast::STATUS_CODEC_INIT_FAILED: - std::move(error_callback) - .Run(base::StringPrintf("%s codec initialization failed.", - is_for_audio ? "Audio" : "Video")); - break; - case media::cast::STATUS_CODEC_RUNTIME_ERROR: - std::move(error_callback) - .Run(base::StringPrintf("%s codec runtime error.", - is_for_audio ? "Audio" : "Video")); - break; - } -} - -void CastSessionDelegate::ReceivePacket( - std::unique_ptr<media::cast::Packet> packet) { - // Do nothing (frees packet) -} diff --git a/chromium/chrome/renderer/media/cast_session_delegate.h b/chromium/chrome/renderer/media/cast_session_delegate.h deleted file mode 100644 index b841c35479c..00000000000 --- a/chromium/chrome/renderer/media/cast_session_delegate.h +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_SESSION_DELEGATE_H_ -#define CHROME_RENDERER_MEDIA_CAST_SESSION_DELEGATE_H_ - -#include <map> -#include <memory> -#include <vector> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "base/threading/thread.h" -#include "base/threading/thread_checker.h" -#include "base/time/default_tick_clock.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/logging_defines.h" - -namespace base { -class DictionaryValue; -class SingleThreadTaskRunner; -class Value; -} // namespace base - -namespace media { - -namespace cast { -class CastEnvironment; -class RawEventSubscriberBundle; - -namespace transport { -class CastTransport; -} // namespace transport -} // namespace cast -} // namespace media - -// Breaks out functionality that is common between CastSessionDelegate and -// CastReceiverSessionDelegate. -class CastSessionDelegateBase { - public: - // TODO(crbug.com/1007641): remove ErrorCallback and rename ErrorOnceCallback - // once all occurrences of base::Callback have been removed. - using ErrorCallback = base::RepeatingCallback<void(const std::string&)>; - using ErrorOnceCallback = base::OnceCallback<void(const std::string&)>; - - CastSessionDelegateBase(); - virtual ~CastSessionDelegateBase(); - - // This will start the session by configuring and creating the Cast transport - // and the Cast sender. - // Must be called before initialization of audio or video. - void StartUDP(const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback); - - protected: - void StatusNotificationCB(ErrorOnceCallback error_callback, - media::cast::CastTransportStatus status); - - virtual void ReceivePacket(std::unique_ptr<media::cast::Packet> packet) = 0; - - base::ThreadChecker thread_checker_; - scoped_refptr<media::cast::CastEnvironment> cast_environment_; - std::unique_ptr<media::cast::CastTransport> cast_transport_; - - // Proxy to the IO message loop. - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - base::WeakPtrFactory<CastSessionDelegateBase> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastSessionDelegateBase); -}; - -// This class hosts CastSender and connects it to audio/video frame input -// and network socket. -// This class is created on the render thread and destroyed on the IO -// thread. All methods are accessible only on the IO thread. -class CastSessionDelegate : public CastSessionDelegateBase { - public: - typedef base::Callback<void(const scoped_refptr< - media::cast::AudioFrameInput>&)> AudioFrameInputAvailableCallback; - typedef base::Callback<void(const scoped_refptr< - media::cast::VideoFrameInput>&)> VideoFrameInputAvailableCallback; - typedef base::Callback<void(std::unique_ptr<base::Value>)> EventLogsCallback; - typedef base::Callback<void(std::unique_ptr<base::DictionaryValue>)> - StatsCallback; - - CastSessionDelegate(); - ~CastSessionDelegate() override; - - void StartUDP(const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback); - - // After calling StartAudio() or StartVideo() encoding of that media will - // begin as soon as data is delivered to its sink, if the second method is - // called the first media will be restarted. It is strongly recommended not to - // deliver any data between calling the two methods. - // It's OK to call only one of the two methods. - // StartUDP must be called before these methods. - void StartAudio(const media::cast::FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - ErrorOnceCallback error_callback); - - void StartVideo( - const media::cast::FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback, - const media::cast::CreateVideoEncodeAcceleratorCallback& create_vea_cb, - const media::cast::CreateVideoEncodeMemoryCallback& - create_video_encode_mem_cb); - - // Start remoting session for one stream. After calling this method, a - // remoting sender will be ready for sending the demuxed stream. StartUDP() - // must be called before calling this method. - void StartRemotingStream(int32_t stream_id, - const media::cast::FrameSenderConfig& config, - ErrorOnceCallback error_callback); - - void ToggleLogging(bool is_audio, bool enable); - void GetEventLogsAndReset(bool is_audio, - const std::string& extra_data, const EventLogsCallback& callback); - void GetStatsAndReset(bool is_audio, const StatsCallback& callback); - - protected: - // Called to report back operational status changes. The first time this is - // called with STATUS_INITIALIZED will result in running the "frame input - // available" callback, to indicate the session is ready to accept incoming - // audio/video frames. If this is called with an error that has halted the - // session, the |error_callback| provided to StartXXX() will be run. This - // method may be called multiple times during the session to indicate codec - // re-initializations are taking place and/or runtime errors have occurred. - void OnOperationalStatusChange(bool is_for_audio, - ErrorOnceCallback error_callback, - media::cast::OperationalStatus result); - - private: - void ReceivePacket(std::unique_ptr<media::cast::Packet> packet) override; - - std::unique_ptr<media::cast::CastSender> cast_sender_; - - AudioFrameInputAvailableCallback audio_frame_input_available_callback_; - VideoFrameInputAvailableCallback video_frame_input_available_callback_; - - std::unique_ptr<media::cast::RawEventSubscriberBundle> event_subscribers_; - - base::WeakPtrFactory<CastSessionDelegate> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastSessionDelegate); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_SESSION_DELEGATE_H_ diff --git a/chromium/chrome/renderer/media/cast_threads.cc b/chromium/chrome/renderer/media/cast_threads.cc deleted file mode 100644 index 823df61f05e..00000000000 --- a/chromium/chrome/renderer/media/cast_threads.cc +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_threads.h" - -#include "base/single_thread_task_runner.h" - -CastThreads::CastThreads() - : audio_encode_thread_("CastAudioEncodeThread"), - video_encode_thread_("CastVideoEncodeThread") { - audio_encode_thread_.Start(); - video_encode_thread_.Start(); -} - -scoped_refptr<base::SingleThreadTaskRunner> -CastThreads::GetAudioEncodeTaskRunner() { - return audio_encode_thread_.task_runner(); -} - -scoped_refptr<base::SingleThreadTaskRunner> -CastThreads::GetVideoEncodeTaskRunner() { - return video_encode_thread_.task_runner(); -} diff --git a/chromium/chrome/renderer/media/cast_threads.h b/chromium/chrome/renderer/media/cast_threads.h deleted file mode 100644 index 01f05552a0c..00000000000 --- a/chromium/chrome/renderer/media/cast_threads.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Manages threads used by Cast Streaming Extensions API. There is a -// singleton object of this class in the renderer. -// -// There are two threads owned by this class: -// 1. Audio encode thread. -// 2. Video encode thread. -// These two threads are started this object is created. - -#ifndef CHROME_RENDERER_MEDIA_CAST_THREADS_H_ -#define CHROME_RENDERER_MEDIA_CAST_THREADS_H_ - -#include "base/lazy_instance.h" -#include "base/macros.h" -#include "base/single_thread_task_runner.h" -#include "base/threading/thread.h" - -class CastThreads { - public: - scoped_refptr<base::SingleThreadTaskRunner> GetAudioEncodeTaskRunner(); - scoped_refptr<base::SingleThreadTaskRunner> GetVideoEncodeTaskRunner(); - - private: - friend struct base::LazyInstanceTraitsBase<CastThreads>; - - CastThreads(); - - base::Thread audio_encode_thread_; - base::Thread video_encode_thread_; - - DISALLOW_COPY_AND_ASSIGN(CastThreads); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_THREADS_H_ diff --git a/chromium/chrome/renderer/media/cast_transport_ipc.cc b/chromium/chrome/renderer/media/cast_transport_ipc.cc deleted file mode 100644 index 06842f3bd02..00000000000 --- a/chromium/chrome/renderer/media/cast_transport_ipc.cc +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_transport_ipc.h" - -#include <memory> -#include <utility> - -#include "base/callback.h" -#include "base/containers/id_map.h" -#include "chrome/common/cast_messages.h" -#include "chrome/renderer/media/cast_ipc_dispatcher.h" -#include "ipc/ipc_channel_proxy.h" -#include "media/cast/cast_sender.h" - -CastTransportIPC::CastTransportIPC( - const net::IPEndPoint& local_end_point, - const net::IPEndPoint& remote_end_point, - std::unique_ptr<base::DictionaryValue> options, - media::cast::PacketReceiverCallback packet_callback, - media::cast::CastTransportStatusCallback status_callback, - BulkRawEventsCallback raw_events_cb) - : channel_id_(-1), - packet_callback_(std::move(packet_callback)), - status_callback_(std::move(status_callback)), - raw_events_callback_(std::move(raw_events_cb)) { - if (CastIPCDispatcher::Get()) { - // TODO(miu): CastIPCDispatcher should be provided as a ctor argument. - channel_id_ = CastIPCDispatcher::Get()->AddSender(this); - Send(new CastHostMsg_New(channel_id_, local_end_point, remote_end_point, - *options)); - } -} - -CastTransportIPC::~CastTransportIPC() { - Send(new CastHostMsg_Delete(channel_id_)); - if (CastIPCDispatcher::Get()) { - CastIPCDispatcher::Get()->RemoveSender(channel_id_); - } -} - -void CastTransportIPC::InitializeStream( - const media::cast::CastTransportRtpConfig& config, - std::unique_ptr<media::cast::RtcpObserver> rtcp_observer) { - if (rtcp_observer) { - DCHECK(clients_.find(config.ssrc) == clients_.end()); - clients_[config.ssrc] = std::move(rtcp_observer); - } - Send(new CastHostMsg_InitializeStream(channel_id_, config)); -} - -void CastTransportIPC::InsertFrame(uint32_t ssrc, - const media::cast::EncodedFrame& frame) { - Send(new CastHostMsg_InsertFrame(channel_id_, ssrc, frame)); -} - -void CastTransportIPC::SendSenderReport( - uint32_t ssrc, - base::TimeTicks current_time, - media::cast::RtpTimeTicks current_time_as_rtp_timestamp) { - Send(new CastHostMsg_SendSenderReport(channel_id_, ssrc, current_time, - current_time_as_rtp_timestamp)); -} - -void CastTransportIPC::CancelSendingFrames( - uint32_t ssrc, - const std::vector<media::cast::FrameId>& frame_ids) { - Send(new CastHostMsg_CancelSendingFrames(channel_id_, ssrc, frame_ids)); -} - -void CastTransportIPC::ResendFrameForKickstart(uint32_t ssrc, - media::cast::FrameId frame_id) { - Send(new CastHostMsg_ResendFrameForKickstart(channel_id_, ssrc, frame_id)); -} - -void CastTransportIPC::AddValidRtpReceiver(uint32_t rtp_sender_ssrc, - uint32_t rtp_receiver_ssrc) { - Send(new CastHostMsg_AddValidRtpReceiver(channel_id_, rtp_sender_ssrc, - rtp_receiver_ssrc)); -} - -void CastTransportIPC::InitializeRtpReceiverRtcpBuilder( - uint32_t rtp_receiver_ssrc, - const media::cast::RtcpTimeData& time_data) { - Send(new CastHostMsg_InitializeRtpReceiverRtcpBuilder( - channel_id_, rtp_receiver_ssrc, time_data)); -} - -void CastTransportIPC::AddCastFeedback( - const media::cast::RtcpCastMessage& cast_message, - base::TimeDelta target_delay) { - Send( - new CastHostMsg_AddCastFeedback(channel_id_, cast_message, target_delay)); -} - -void CastTransportIPC::AddPli(const media::cast::RtcpPliMessage& pli_message) { - Send(new CastHostMsg_AddPli(channel_id_, pli_message)); -} - -void CastTransportIPC::AddRtcpEvents( - const media::cast::ReceiverRtcpEventSubscriber::RtcpEvents& rtcp_events) { - Send(new CastHostMsg_AddRtcpEvents(channel_id_, rtcp_events)); -} - -void CastTransportIPC::AddRtpReceiverReport( - const media::cast::RtcpReportBlock& rtp_receiver_report_block) { - Send(new CastHostMsg_AddRtpReceiverReport(channel_id_, - rtp_receiver_report_block)); -} - -void CastTransportIPC::SendRtcpFromRtpReceiver() { - Send(new CastHostMsg_SendRtcpFromRtpReceiver(channel_id_)); -} - -void CastTransportIPC::OnNotifyStatusChange( - media::cast::CastTransportStatus status) { - status_callback_.Run(status); -} - -void CastTransportIPC::OnRawEvents( - const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events) { - // Note: Casting away const to avoid having to copy all the data elements. As - // the only consumer of this data in the IPC message, mutating the inputs - // should be acceptable. Just nod and blame the interface we were given here. - std::unique_ptr<std::vector<media::cast::FrameEvent>> taken_frame_events( - new std::vector<media::cast::FrameEvent>()); - taken_frame_events->swap( - const_cast<std::vector<media::cast::FrameEvent>&>(frame_events)); - std::unique_ptr<std::vector<media::cast::PacketEvent>> taken_packet_events( - new std::vector<media::cast::PacketEvent>()); - taken_packet_events->swap( - const_cast<std::vector<media::cast::PacketEvent>&>(packet_events)); - raw_events_callback_.Run(std::move(taken_frame_events), - std::move(taken_packet_events)); -} - -void CastTransportIPC::OnRtt(uint32_t rtp_sender_ssrc, base::TimeDelta rtt) { - auto it = clients_.find(rtp_sender_ssrc); - if (it == clients_.end()) { - LOG(ERROR) << "Received RTT report for unknown SSRC: " << rtp_sender_ssrc; - return; - } - it->second->OnReceivedRtt(rtt); -} - -void CastTransportIPC::OnRtcpCastMessage( - uint32_t rtp_sender_ssrc, - const media::cast::RtcpCastMessage& cast_message) { - auto it = clients_.find(rtp_sender_ssrc); - if (it == clients_.end()) { - LOG(ERROR) << "Received cast message for unknown SSRC: " << rtp_sender_ssrc; - return; - } - it->second->OnReceivedCastMessage(cast_message); -} - -void CastTransportIPC::OnReceivedPli(uint32_t rtp_sender_ssrc) { - auto it = clients_.find(rtp_sender_ssrc); - if (it == clients_.end()) { - LOG(ERROR) << "Received picture loss indicator for unknown SSRC: " - << rtp_sender_ssrc; - return; - } - it->second->OnReceivedPli(); -} - -void CastTransportIPC::OnReceivedPacket(const media::cast::Packet& packet) { - if (!packet_callback_.is_null()) { - // TODO(hubbe): Perhaps an non-ownership-transferring cb here? - std::unique_ptr<media::cast::Packet> packet_copy( - new media::cast::Packet(packet)); - packet_callback_.Run(std::move(packet_copy)); - } else { - DVLOG(1) << "CastIPCDispatcher::OnReceivedPacket no packet callback yet."; - } -} - -void CastTransportIPC::Send(IPC::Message* message) { - if (CastIPCDispatcher::Get() && channel_id_ != -1) { - CastIPCDispatcher::Get()->Send(message); - } else { - delete message; - } -} diff --git a/chromium/chrome/renderer/media/cast_transport_ipc.h b/chromium/chrome/renderer/media/cast_transport_ipc.h deleted file mode 100644 index 3ee563bd636..00000000000 --- a/chromium/chrome/renderer/media/cast_transport_ipc.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_TRANSPORT_IPC_H_ -#define CHROME_RENDERER_MEDIA_CAST_TRANSPORT_IPC_H_ - -#include <stdint.h> - -#include <map> -#include <memory> - -#include "base/macros.h" -#include "base/threading/thread_task_runner_handle.h" -#include "ipc/ipc_channel_proxy.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/net/cast_transport.h" - -// This implementation of the CastTransport interface communicates with the -// browser process over IPC and relays all calls to/from the cast transport to -// the browser process. The primary reason for this arrangement is to give the -// renderer less direct control over the UDP sockets. -class CastTransportIPC : public media::cast::CastTransport { - public: - // Runs when a raw event completes. - using BulkRawEventsCallback = base::RepeatingCallback<void( - std::unique_ptr<std::vector<media::cast::FrameEvent>>, - std::unique_ptr<std::vector<media::cast::PacketEvent>>)>; - - CastTransportIPC(const net::IPEndPoint& local_end_point, - const net::IPEndPoint& remote_end_point, - std::unique_ptr<base::DictionaryValue> options, - media::cast::PacketReceiverCallback packet_callback, - media::cast::CastTransportStatusCallback status_callback, - BulkRawEventsCallback raw_events_cb); - - ~CastTransportIPC() override; - - // media::cast::CastTransport implementation. - void InitializeStream( - const media::cast::CastTransportRtpConfig& config, - std::unique_ptr<media::cast::RtcpObserver> rtcp_observer) override; - void InsertFrame(uint32_t ssrc, - const media::cast::EncodedFrame& frame) override; - void SendSenderReport( - uint32_t ssrc, - base::TimeTicks current_time, - media::cast::RtpTimeTicks current_time_as_rtp_timestamp) override; - void CancelSendingFrames( - uint32_t ssrc, - const std::vector<media::cast::FrameId>& frame_ids) override; - void ResendFrameForKickstart(uint32_t ssrc, - media::cast::FrameId frame_id) override; - void AddValidRtpReceiver(uint32_t rtp_sender_ssrc, - uint32_t rtp_receiver_ssrc) override; - void InitializeRtpReceiverRtcpBuilder( - uint32_t rtp_receiver_ssrc, - const media::cast::RtcpTimeData& time_data) override; - void AddCastFeedback(const media::cast::RtcpCastMessage& cast_message, - base::TimeDelta target_delay) override; - void AddPli(const media::cast::RtcpPliMessage& pli_message) override; - void AddRtcpEvents(const media::cast::ReceiverRtcpEventSubscriber::RtcpEvents& - rtcp_events) override; - void AddRtpReceiverReport( - const media::cast::RtcpReportBlock& rtp_receiver_report_block) override; - void SendRtcpFromRtpReceiver() override; - void SetOptions(const base::DictionaryValue& options) final {} - void OnNotifyStatusChange(media::cast::CastTransportStatus status); - void OnRawEvents(const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events); - void OnRtt(uint32_t rtp_sender_ssrc, base::TimeDelta rtt); - void OnRtcpCastMessage(uint32_t rtp_sender_ssrc, - const media::cast::RtcpCastMessage& cast_message); - void OnReceivedPli(uint32_t rtp_sender_ssrc); - void OnReceivedPacket(const media::cast::Packet& packet); - - private: - void Send(IPC::Message* message); - - int32_t channel_id_; - media::cast::PacketReceiverCallback packet_callback_; - media::cast::CastTransportStatusCallback status_callback_; - const BulkRawEventsCallback raw_events_callback_; - using ClientMap = - std::map<uint32_t, std::unique_ptr<media::cast::RtcpObserver>>; - ClientMap clients_; - - DISALLOW_COPY_AND_ASSIGN(CastTransportIPC); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_TRANSPORT_IPC_H_ diff --git a/chromium/chrome/renderer/media/cast_udp_transport.cc b/chromium/chrome/renderer/media/cast_udp_transport.cc deleted file mode 100644 index a8302a8a0fa..00000000000 --- a/chromium/chrome/renderer/media/cast_udp_transport.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_udp_transport.h" - -#include <memory> -#include <utility> - -#include "base/memory/ptr_util.h" -#include "base/values.h" -#include "chrome/renderer/media/cast_session.h" - -CastUdpTransport::CastUdpTransport(const scoped_refptr<CastSession>& session) - : cast_session_(session), options_(new base::DictionaryValue) {} - -CastUdpTransport::~CastUdpTransport() { -} - -void CastUdpTransport::SetDestination( - const net::IPEndPoint& remote_address, - const CastSessionDelegate::ErrorCallback& error_callback) { - DVLOG(1) << "CastUdpTransport::SetDestination = " - << remote_address.ToString(); - remote_address_ = remote_address; - cast_session_->StartUDP( - remote_address, base::WrapUnique(options_->DeepCopy()), error_callback); -} - -void CastUdpTransport::SetOptions( - std::unique_ptr<base::DictionaryValue> options) { - options_ = std::move(options); -} diff --git a/chromium/chrome/renderer/media/cast_udp_transport.h b/chromium/chrome/renderer/media/cast_udp_transport.h deleted file mode 100644 index 726edbe5c67..00000000000 --- a/chromium/chrome/renderer/media/cast_udp_transport.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_UDP_TRANSPORT_H_ -#define CHROME_RENDERER_MEDIA_CAST_UDP_TRANSPORT_H_ - -#include <memory> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "chrome/renderer/media/cast_session_delegate.h" -#include "net/base/ip_endpoint.h" - -namespace base { -class DictionaryValue; -} // namespace base - -class CastSession; - -// This class represents the transport mechanism used by Cast RTP streams -// to connect to a remote client. It specifies the destination address -// and network protocol used to send Cast RTP streams. -class CastUdpTransport { - public: - explicit CastUdpTransport(const scoped_refptr<CastSession>& session); - virtual ~CastUdpTransport(); - - // Specify the remote IP address and port. - void SetDestination(const net::IPEndPoint& remote_address, - const CastSessionDelegate::ErrorCallback& error_callback); - - // Set options. - void SetOptions(std::unique_ptr<base::DictionaryValue> options); - - private: - const scoped_refptr<CastSession> cast_session_; - net::IPEndPoint remote_address_; - std::unique_ptr<base::DictionaryValue> options_; - base::WeakPtrFactory<CastUdpTransport> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastUdpTransport); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_UDP_TRANSPORT_H_ diff --git a/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc b/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc index 2b35d6d399a..70da74b25e7 100644 --- a/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc +++ b/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc @@ -6,22 +6,60 @@ #include <utility> +#include "base/metrics/field_trial_params.h" #include "content/public/renderer/render_frame.h" +#include "media/base/audio_bus.h" +#include "media/base/audio_parameters.h" +#include "media/base/bind_to_current_loop.h" +#include "media/base/channel_mixer.h" +#include "media/base/media_switches.h" #include "media/mojo/mojom/media_types.mojom.h" #include "third_party/blink/public/common/browser_interface_broker_proxy.h" +#include "third_party/blink/public/platform/web_string.h" +#include "third_party/blink/public/web/web_frame.h" +#include "third_party/blink/public/web/web_local_frame.h" + +// Get the list of blocked URLs defined by the Finch experiment parameter. These +// websites provide captions by default and thus do not require the live caption +// feature. +std::vector<std::string> GetBlockedURLs() { + return base::SplitString(base::GetFieldTrialParamValueByFeature( + media::kLiveCaption, "blocked_websites"), + ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); +} ChromeSpeechRecognitionClient::ChromeSpeechRecognitionClient( - content::RenderFrame* render_frame) { + content::RenderFrame* render_frame, + media::SpeechRecognitionClient::OnReadyCallback callback) + : on_ready_callback_(std::move(callback)), blocked_urls_(GetBlockedURLs()) { mojo::PendingReceiver<media::mojom::SpeechRecognitionContext> speech_recognition_context_receiver = speech_recognition_context_.BindNewPipeAndPassReceiver(); speech_recognition_context_->BindRecognizer( speech_recognition_recognizer_.BindNewPipeAndPassReceiver(), - speech_recognition_client_receiver_.BindNewPipeAndPassRemote()); + speech_recognition_client_receiver_.BindNewPipeAndPassRemote(), + base::BindOnce(&ChromeSpeechRecognitionClient::OnRecognizerBound, + base::Unretained(this))); + render_frame->GetBrowserInterfaceBroker()->GetInterface( std::move(speech_recognition_context_receiver)); render_frame->GetBrowserInterfaceBroker()->GetInterface( caption_host_.BindNewPipeAndPassReceiver()); + is_website_blocked_ = IsUrlBlocked( + render_frame->GetWebFrame()->GetSecurityOrigin().ToString().Utf8()); + + send_audio_callback_ = media::BindToCurrentLoop(base::BindRepeating( + &ChromeSpeechRecognitionClient::SendAudioToSpeechRecognitionService, + weak_factory_.GetWeakPtr())); +} + +void ChromeSpeechRecognitionClient::OnRecognizerBound( + bool is_multichannel_supported) { + is_multichannel_supported_ = is_multichannel_supported; + is_recognizer_bound_ = true; + + if (on_ready_callback_) + std::move(on_ready_callback_).Run(); } ChromeSpeechRecognitionClient::~ChromeSpeechRecognitionClient() = default; @@ -29,21 +67,85 @@ ChromeSpeechRecognitionClient::~ChromeSpeechRecognitionClient() = default; void ChromeSpeechRecognitionClient::AddAudio( scoped_refptr<media::AudioBuffer> buffer) { DCHECK(buffer); - if (IsSpeechRecognitionAvailable()) { - speech_recognition_recognizer_->SendAudioToSpeechRecognitionService( - ConvertToAudioDataS16(std::move(buffer))); - } + send_audio_callback_.Run(ConvertToAudioDataS16(std::move(buffer))); +} + +void ChromeSpeechRecognitionClient::AddAudio( + std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout) { + DCHECK(audio_bus); + send_audio_callback_.Run( + ConvertToAudioDataS16(std::move(audio_bus), sample_rate, channel_layout)); } bool ChromeSpeechRecognitionClient::IsSpeechRecognitionAvailable() { - return speech_recognition_recognizer_.is_bound() && - speech_recognition_recognizer_.is_connected(); + // TODO(evliu): Check if SODA is available. + return !is_website_blocked_ && is_browser_requesting_transcription_ && + is_recognizer_bound_; +} + +// The OnReadyCallback is set by the owner of |this| and is executed when speech +// recognition becomes available. Setting the callback will override any +// existing callback. +void ChromeSpeechRecognitionClient::SetOnReadyCallback( + SpeechRecognitionClient::OnReadyCallback callback) { + on_ready_callback_ = std::move(callback); + + // Immediately run the callback if speech recognition is already available. + if (IsSpeechRecognitionAvailable() && on_ready_callback_) + std::move(on_ready_callback_).Run(); } void ChromeSpeechRecognitionClient::OnSpeechRecognitionRecognitionEvent( media::mojom::SpeechRecognitionResultPtr result) { - caption_host_->OnTranscription(chrome::mojom::TranscriptionResult::New( - result->transcription, result->is_final)); + caption_host_->OnTranscription( + chrome::mojom::TranscriptionResult::New(result->transcription, + result->is_final), + base::BindOnce(&ChromeSpeechRecognitionClient::OnTranscriptionCallback, + base::Unretained(this))); +} + +void ChromeSpeechRecognitionClient::OnTranscriptionCallback(bool success) { + is_browser_requesting_transcription_ = success; +} + +void ChromeSpeechRecognitionClient::CopyBufferToTempAudioBus( + const media::AudioBuffer& buffer) { + if (!temp_audio_bus_ || + buffer.channel_count() != temp_audio_bus_->channels() || + buffer.frame_count() != temp_audio_bus_->frames()) { + temp_audio_bus_ = + media::AudioBus::Create(buffer.channel_count(), buffer.frame_count()); + } + + buffer.ReadFrames(buffer.frame_count(), + /* source_frame_offset */ 0, /* dest_frame_offset */ 0, + temp_audio_bus_.get()); +} + +void ChromeSpeechRecognitionClient::ResetChannelMixer( + int frame_count, + media::ChannelLayout channel_layout) { + if (!monaural_audio_bus_ || frame_count != monaural_audio_bus_->frames()) { + monaural_audio_bus_ = + media::AudioBus::Create(1 /* channels */, frame_count); + } + + if (channel_layout != channel_layout_) { + channel_layout_ = channel_layout; + channel_mixer_ = std::make_unique<media::ChannelMixer>( + channel_layout, media::CHANNEL_LAYOUT_MONO); + } +} + +void ChromeSpeechRecognitionClient::SendAudioToSpeechRecognitionService( + media::mojom::AudioDataS16Ptr audio_data) { + DCHECK(audio_data); + if (IsSpeechRecognitionAvailable()) { + speech_recognition_recognizer_->SendAudioToSpeechRecognitionService( + std::move(audio_data)); + } } media::mojom::AudioDataS16Ptr @@ -58,6 +160,21 @@ ChromeSpeechRecognitionClient::ConvertToAudioDataS16( signed_buffer->frame_count = buffer->frame_count(); signed_buffer->sample_rate = buffer->sample_rate(); + // If multichannel audio is not supported by the speech recognition service, + // mix the channels into a monaural channel before converting it. + if (buffer->channel_count() > 1 && !is_multichannel_supported_) { + signed_buffer->channel_count = 1; + CopyBufferToTempAudioBus(*buffer); + ResetChannelMixer(buffer->frame_count(), buffer->channel_layout()); + signed_buffer->data.resize(buffer->frame_count()); + channel_mixer_->Transform(temp_audio_bus_.get(), monaural_audio_bus_.get()); + monaural_audio_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>( + monaural_audio_bus_->frames(), &signed_buffer->data[0]); + return signed_buffer; + } + + // If the audio is already in the interleaved signed int 16 format, directly + // assign it to the buffer. if (buffer->sample_format() == media::SampleFormat::kSampleFormatS16) { int16_t* audio_data = reinterpret_cast<int16_t*>(buffer->channel_data()[0]); signed_buffer->data.assign( @@ -67,20 +184,48 @@ ChromeSpeechRecognitionClient::ConvertToAudioDataS16( } // Convert the raw audio to the interleaved signed int 16 sample type. - if (!temp_audio_bus_ || - buffer->channel_count() != temp_audio_bus_->channels() || - buffer->frame_count() != temp_audio_bus_->frames()) { - temp_audio_bus_ = - media::AudioBus::Create(buffer->channel_count(), buffer->frame_count()); - } - - buffer->ReadFrames(buffer->frame_count(), - /* source_frame_offset */ 0, /* dest_frame_offset */ 0, - temp_audio_bus_.get()); - + CopyBufferToTempAudioBus(*buffer); signed_buffer->data.resize(buffer->frame_count() * buffer->channel_count()); temp_audio_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>( temp_audio_bus_->frames(), &signed_buffer->data[0]); return signed_buffer; } + +media::mojom::AudioDataS16Ptr +ChromeSpeechRecognitionClient::ConvertToAudioDataS16( + std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout) { + DCHECK_GT(audio_bus->frames(), 0); + DCHECK_GT(audio_bus->channels(), 0); + + auto signed_buffer = media::mojom::AudioDataS16::New(); + signed_buffer->channel_count = audio_bus->channels(); + signed_buffer->frame_count = audio_bus->frames(); + signed_buffer->sample_rate = sample_rate; + + // If multichannel audio is not supported by the speech recognition service, + // mix the channels into a monaural channel before converting it. + if (audio_bus->channels() > 1 && !is_multichannel_supported_) { + signed_buffer->channel_count = 1; + ResetChannelMixer(audio_bus->frames(), channel_layout); + signed_buffer->data.resize(audio_bus->frames()); + + channel_mixer_->Transform(audio_bus.get(), monaural_audio_bus_.get()); + monaural_audio_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>( + monaural_audio_bus_->frames(), &signed_buffer->data[0]); + + return signed_buffer; + } + + signed_buffer->data.resize(audio_bus->frames() * audio_bus->channels()); + audio_bus->ToInterleaved<media::SignedInt16SampleTypeTraits>( + audio_bus->frames(), &signed_buffer->data[0]); + + return signed_buffer; +} + +bool ChromeSpeechRecognitionClient::IsUrlBlocked(const std::string& url) const { + return blocked_urls_.find(url) != blocked_urls_.end(); +} diff --git a/chromium/chrome/renderer/media/chrome_speech_recognition_client.h b/chromium/chrome/renderer/media/chrome_speech_recognition_client.h index 5f08b42cfd9..df0e0574a87 100644 --- a/chromium/chrome/renderer/media/chrome_speech_recognition_client.h +++ b/chromium/chrome/renderer/media/chrome_speech_recognition_client.h @@ -8,6 +8,8 @@ #include <memory> #include <string> +#include "base/containers/flat_set.h" +#include "base/memory/weak_ptr.h" #include "chrome/common/caption.mojom.h" #include "media/base/audio_buffer.h" #include "media/base/speech_recognition_client.h" @@ -19,11 +21,21 @@ namespace content { class RenderFrame; } // namespace content +namespace media { +class AudioBus; +class ChannelMixer; +} // namespace media + class ChromeSpeechRecognitionClient : public media::SpeechRecognitionClient, public media::mojom::SpeechRecognitionRecognizerClient { public: - explicit ChromeSpeechRecognitionClient(content::RenderFrame* render_frame); + using SendAudioToSpeechRecognitionServiceCallback = + base::RepeatingCallback<void(media::mojom::AudioDataS16Ptr audio_data)>; + + explicit ChromeSpeechRecognitionClient( + content::RenderFrame* render_frame, + media::SpeechRecognitionClient::OnReadyCallback callback); ChromeSpeechRecognitionClient(const ChromeSpeechRecognitionClient&) = delete; ChromeSpeechRecognitionClient& operator=( const ChromeSpeechRecognitionClient&) = delete; @@ -31,16 +43,51 @@ class ChromeSpeechRecognitionClient // media::SpeechRecognitionClient void AddAudio(scoped_refptr<media::AudioBuffer> buffer) override; + void AddAudio(std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout) override; bool IsSpeechRecognitionAvailable() override; + void SetOnReadyCallback( + SpeechRecognitionClient::OnReadyCallback callback) override; + + // Callback executed when the recognizer is bound. Sets the flag indicating + // whether the speech recognition service supports multichannel audio. + void OnRecognizerBound(bool is_multichannel_supported); // media::mojom::SpeechRecognitionRecognizerClient void OnSpeechRecognitionRecognitionEvent( media::mojom::SpeechRecognitionResultPtr result) override; private: + void SendAudioToSpeechRecognitionService( + media::mojom::AudioDataS16Ptr audio_data); + media::mojom::AudioDataS16Ptr ConvertToAudioDataS16( scoped_refptr<media::AudioBuffer> buffer); + // Called as a response to sending a transcription to the browser. + void OnTranscriptionCallback(bool success); + + media::mojom::AudioDataS16Ptr ConvertToAudioDataS16( + std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout); + + // Recreates the temporary audio bus if the frame count or channel count + // changed and reads the frames from the buffer into the temporary audio bus. + void CopyBufferToTempAudioBus(const media::AudioBuffer& buffer); + + // Resets the temporary monaural audio bus and the channel mixer used to + // combine multiple audio channels. + void ResetChannelMixer(int frame_count, media::ChannelLayout channel_layout); + + bool IsUrlBlocked(const std::string& url) const; + + media::SpeechRecognitionClient::OnReadyCallback on_ready_callback_; + + // Sends audio to the speech recognition thread on the renderer thread. + SendAudioToSpeechRecognitionServiceCallback send_audio_callback_; + mojo::Remote<media::mojom::SpeechRecognitionContext> speech_recognition_context_; mojo::Remote<media::mojom::SpeechRecognitionRecognizer> @@ -49,9 +96,33 @@ class ChromeSpeechRecognitionClient speech_recognition_client_receiver_{this}; mojo::Remote<chrome::mojom::CaptionHost> caption_host_; + bool is_website_blocked_ = false; + const base::flat_set<std::string> blocked_urls_; + // The temporary audio bus used to convert the raw audio to the appropriate // format. std::unique_ptr<media::AudioBus> temp_audio_bus_; + + // Whether the browser is still requesting transcriptions. + bool is_browser_requesting_transcription_ = true; + + bool is_recognizer_bound_ = false; + + // The temporary audio bus used to mix multichannel audio into a single + // channel. + std::unique_ptr<media::AudioBus> monaural_audio_bus_; + + std::unique_ptr<media::ChannelMixer> channel_mixer_; + + // The layout used to instantiate the channel mixer. + media::ChannelLayout channel_layout_ = + media::ChannelLayout::CHANNEL_LAYOUT_NONE; + + // A flag indicating whether the speech recognition service supports + // multichannel audio. + bool is_multichannel_supported_ = false; + + base::WeakPtrFactory<ChromeSpeechRecognitionClient> weak_factory_{this}; }; #endif // CHROME_RENDERER_MEDIA_CHROME_SPEECH_RECOGNITION_CLIENT_H_ diff --git a/chromium/chrome/renderer/media/media_feeds.cc b/chromium/chrome/renderer/media/media_feeds.cc index cdbc1995457..fae6fdeb893 100644 --- a/chromium/chrome/renderer/media/media_feeds.cc +++ b/chromium/chrome/renderer/media/media_feeds.cc @@ -45,14 +45,9 @@ base::Optional<GURL> MediaFeeds::GetMediaFeedURL(content::RenderFrame* frame) { if (!elem.HasHTMLTagName("link")) continue; - // The <link> rel must be feed. + // The <link> rel must be media-feed. std::string rel = elem.GetAttribute("rel").Utf8(); - if (!base::LowerCaseEqualsASCII(rel, "feed")) - continue; - - // The <link> type must the JSON+LD mime type. - std::string type = elem.GetAttribute("type").Utf8(); - if (!base::LowerCaseEqualsASCII(type, "application/ld+json")) + if (!base::LowerCaseEqualsASCII(rel, "media-feed")) continue; WebString href = elem.GetAttribute("href"); diff --git a/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc b/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc index b8b24d4ebb9..d2495adc221 100644 --- a/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc +++ b/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc @@ -82,6 +82,7 @@ void WebRtcLoggingAgentImpl::AddReceiver( void WebRtcLoggingAgentImpl::Start( mojo::PendingRemote<mojom::WebRtcLoggingClient> pending_client) { // We only support one client at a time. OK to drop any existing client. + client_.reset(); client_.Bind(std::move(pending_client)); WebRtcLogMessageDelegateImpl::GetInstance()->Start(base::BindRepeating( |