diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-12 14:27:29 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:35:20 +0000 |
commit | c30a6232df03e1efbd9f3b226777b07e087a1122 (patch) | |
tree | e992f45784689f373bcc38d1b79a239ebe17ee23 /chromium/third_party/webrtc/call | |
parent | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (diff) | |
download | qtwebengine-chromium-85-based.tar.gz |
BASELINE: Update Chromium to 85.0.4183.14085-based
Change-Id: Iaa42f4680837c57725b1344f108c0196741f6057
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/third_party/webrtc/call')
60 files changed, 2896 insertions, 1335 deletions
diff --git a/chromium/third_party/webrtc/call/BUILD.gn b/chromium/third_party/webrtc/call/BUILD.gn index a9037c3819f..5f7c603c8d1 100644 --- a/chromium/third_party/webrtc/call/BUILD.gn +++ b/chromium/third_party/webrtc/call/BUILD.gn @@ -39,6 +39,7 @@ rtc_library("call_interfaces") { "../api:rtp_parameters", "../api:scoped_refptr", "../api:transport_api", + "../api/adaptation:resource_adaptation_api", "../api/audio:audio_mixer_api", "../api/audio_codecs:audio_codecs_api", "../api/crypto:frame_decryptor_interface", @@ -61,8 +62,8 @@ rtc_library("call_interfaces") { "../rtc_base:checks", "../rtc_base:rtc_base_approved", "../rtc_base/network:sent_packet", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_source_set("audio_sender_interface") { @@ -100,6 +101,8 @@ rtc_library("rtp_interfaces") { "../modules/rtp_rtcp:rtp_rtcp_format", "../rtc_base:checks", "../rtc_base:rtc_base_approved", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/types:optional", ] @@ -128,8 +131,8 @@ rtc_library("rtp_receiver") { "../modules/rtp_rtcp:rtp_rtcp_format", "../rtc_base:checks", "../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("rtp_sender") { @@ -171,6 +174,7 @@ rtc_library("rtp_sender") { "../modules/rtp_rtcp:rtp_rtcp_format", "../modules/rtp_rtcp:rtp_video_header", "../modules/utility", + "../modules/video_coding:chain_diff_calculator", "../modules/video_coding:codec_globals_headers", "../modules/video_coding:frame_dependencies_calculator", "../modules/video_coding:video_codec_interface", @@ -180,6 +184,8 @@ rtc_library("rtp_sender") { "../rtc_base:rtc_base_approved", "../rtc_base:rtc_task_queue", "../rtc_base/task_utils:repeating_task", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings:strings", @@ -202,8 +208,8 @@ rtc_library("bitrate_configurator") { "../api/units:data_rate", "../rtc_base:checks", "../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("bitrate_allocator") { @@ -223,8 +229,8 @@ rtc_library("bitrate_allocator") { "../system_wrappers", "../system_wrappers:field_trial", "../system_wrappers:metrics", - "//third_party/abseil-cpp/absl/algorithm:container", ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] } rtc_library("call") { @@ -279,14 +285,15 @@ rtc_library("call") { "../rtc_base:safe_minmax", "../rtc_base/experiments:field_trial_parser", "../rtc_base/network:sent_packet", - "../rtc_base/synchronization:rw_lock_wrapper", "../rtc_base/synchronization:sequence_checker", + "../rtc_base/task_utils:pending_task_safety_flag", "../system_wrappers", "../system_wrappers:field_trial", "../system_wrappers:metrics", "../video", - "//third_party/abseil-cpp/absl/types:optional", + "adaptation:resource_adaptation", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("video_stream_api") { @@ -301,7 +308,9 @@ rtc_library("video_stream_api") { "../api:frame_transformer_interface", "../api:rtp_headers", "../api:rtp_parameters", + "../api:scoped_refptr", "../api:transport_api", + "../api/adaptation:resource_adaptation_api", "../api/crypto:frame_decryptor_interface", "../api/crypto:frame_encryptor_interface", "../api/crypto:options", @@ -315,8 +324,8 @@ rtc_library("video_stream_api") { "../modules/rtp_rtcp:rtp_rtcp_format", "../rtc_base:checks", "../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("simulated_network") { @@ -333,8 +342,8 @@ rtc_library("simulated_network") { "../rtc_base:checks", "../rtc_base:rtc_base_approved", "../rtc_base/synchronization:sequence_checker", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_source_set("simulated_packet_receiver") { @@ -402,7 +411,9 @@ if (rtc_include_tests) { "../api/audio_codecs:builtin_audio_decoder_factory", "../api/rtc_event_log", "../api/task_queue:default_task_queue_factory", + "../api/test/video:function_video_factory", "../api/transport:field_trial_based_config", + "../api/video:builtin_video_bitrate_allocator_factory", "../api/video:video_frame", "../api/video:video_rtp_headers", "../audio", @@ -436,12 +447,16 @@ if (rtc_include_tests) { "../test:video_test_common", "../test/time_controller:time_controller", "../video", + "adaptation:resource_adaptation_test_utilities", "//test/scenario:scenario", "//testing/gmock", "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", ] } @@ -494,8 +509,8 @@ if (rtc_include_tests) { "../test:video_test_common", "../video", "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag" ] } # TODO(eladalon): This should be moved, as with the TODO for |rtp_interfaces|. @@ -553,7 +568,7 @@ if (rtc_include_tests) { "../system_wrappers", "../test:test_support", "//testing/gtest", - "//third_party/abseil-cpp/absl/algorithm:container", ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container" ] } } diff --git a/chromium/third_party/webrtc/call/adaptation/BUILD.gn b/chromium/third_party/webrtc/call/adaptation/BUILD.gn index 2a6933ebd57..055fc437823 100644 --- a/chromium/third_party/webrtc/call/adaptation/BUILD.gn +++ b/chromium/third_party/webrtc/call/adaptation/BUILD.gn @@ -10,10 +10,14 @@ import("../../webrtc.gni") rtc_library("resource_adaptation") { sources = [ + "adaptation_constraint.cc", + "adaptation_constraint.h", + "adaptation_listener.cc", + "adaptation_listener.h", + "broadcast_resource_listener.cc", + "broadcast_resource_listener.h", "encoder_settings.cc", "encoder_settings.h", - "resource.cc", - "resource.h", "resource_adaptation_processor.cc", "resource_adaptation_processor.h", "resource_adaptation_processor_interface.cc", @@ -30,6 +34,8 @@ rtc_library("resource_adaptation") { deps = [ "../../api:rtp_parameters", "../../api:scoped_refptr", + "../../api/adaptation:resource_adaptation_api", + "../../api/task_queue:task_queue", "../../api/video:video_adaptation", "../../api/video:video_frame", "../../api/video:video_stream_encoder", @@ -40,6 +46,9 @@ rtc_library("resource_adaptation") { "../../rtc_base:rtc_task_queue", "../../rtc_base/experiments:balanced_degradation_settings", "../../rtc_base/synchronization:sequence_checker", + "../../rtc_base/task_utils:to_queued_task", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/types:optional", ] @@ -50,6 +59,7 @@ if (rtc_include_tests) { testonly = true sources = [ + "broadcast_resource_listener_unittest.cc", "resource_adaptation_processor_unittest.cc", "resource_unittest.cc", "video_source_restrictions_unittest.cc", @@ -60,35 +70,48 @@ if (rtc_include_tests) { ":resource_adaptation", ":resource_adaptation_test_utilities", "../../api:scoped_refptr", + "../../api/adaptation:resource_adaptation_api", "../../api/task_queue:default_task_queue_factory", "../../api/task_queue:task_queue", "../../api/video:video_adaptation", "../../api/video_codecs:video_codecs_api", "../../rtc_base:checks", + "../../rtc_base:gunit_helpers", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_task_queue", "../../rtc_base:task_queue_for_test", "../../test:field_trial", "../../test:rtc_expect_death", "../../test:test_support", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_source_set("resource_adaptation_test_utilities") { testonly = true sources = [ + "test/fake_adaptation_constraint.cc", + "test/fake_adaptation_constraint.h", + "test/fake_adaptation_listener.cc", + "test/fake_adaptation_listener.h", "test/fake_frame_rate_provider.cc", "test/fake_frame_rate_provider.h", "test/fake_resource.cc", "test/fake_resource.h", + "test/mock_resource_listener.h", ] deps = [ ":resource_adaptation", + "../../api:scoped_refptr", + "../../api/adaptation:resource_adaptation_api", + "../../api/task_queue:task_queue", "../../api/video:video_stream_encoder", "../../rtc_base:rtc_base_approved", + "../../rtc_base/synchronization:sequence_checker", + "../../rtc_base/task_utils:to_queued_task", "../../test:test_support", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } } diff --git a/chromium/third_party/webrtc/call/adaptation/adaptation_constraint.cc b/chromium/third_party/webrtc/call/adaptation/adaptation_constraint.cc new file mode 100644 index 00000000000..d62bb74f87e --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/adaptation_constraint.cc @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/adaptation_constraint.h" + +namespace webrtc { + +AdaptationConstraint::~AdaptationConstraint() {} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/adaptation_constraint.h b/chromium/third_party/webrtc/call/adaptation/adaptation_constraint.h new file mode 100644 index 00000000000..9ff15d6b860 --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/adaptation_constraint.h @@ -0,0 +1,43 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_ADAPTATION_CONSTRAINT_H_ +#define CALL_ADAPTATION_ADAPTATION_CONSTRAINT_H_ + +#include <string> + +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" + +namespace webrtc { + +// Adaptation constraints have the ability to prevent applying a proposed +// adaptation (expressed as restrictions before/after adaptation). +class AdaptationConstraint { + public: + virtual ~AdaptationConstraint(); + + virtual std::string Name() const = 0; + + // TODO(https://crbug.com/webrtc/11172): When we have multi-stream adaptation + // support, this interface needs to indicate which stream the adaptation + // applies to. + virtual bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr<Resource> reason_resource) const = 0; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_ADAPTATION_CONSTRAINT_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/adaptation_listener.cc b/chromium/third_party/webrtc/call/adaptation/adaptation_listener.cc new file mode 100644 index 00000000000..acc1564f770 --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/adaptation_listener.cc @@ -0,0 +1,17 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/adaptation_listener.h" + +namespace webrtc { + +AdaptationListener::~AdaptationListener() {} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/adaptation_listener.h b/chromium/third_party/webrtc/call/adaptation/adaptation_listener.h new file mode 100644 index 00000000000..4a96baef8e0 --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/adaptation_listener.h @@ -0,0 +1,41 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_ADAPTATION_LISTENER_H_ +#define CALL_ADAPTATION_ADAPTATION_LISTENER_H_ + +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" +#include "call/adaptation/video_source_restrictions.h" +#include "call/adaptation/video_stream_input_state.h" + +namespace webrtc { + +// TODO(hbos): Can this be consolidated with +// ResourceAdaptationProcessorListener::OnVideoSourceRestrictionsUpdated()? Both +// listen to adaptations being applied, but on different layers with different +// arguments. +class AdaptationListener { + public: + virtual ~AdaptationListener(); + + // TODO(https://crbug.com/webrtc/11172): When we have multi-stream adaptation + // support, this interface needs to indicate which stream the adaptation + // applies to. + virtual void OnAdaptationApplied( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr<Resource> reason_resource) = 0; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_ADAPTATION_LISTENER_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener.cc b/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener.cc new file mode 100644 index 00000000000..2a4d8cab09d --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener.cc @@ -0,0 +1,120 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/broadcast_resource_listener.h" + +#include <algorithm> +#include <string> +#include <utility> + +#include "rtc_base/checks.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/ref_counted_object.h" + +namespace webrtc { + +// The AdapterResource redirects resource usage measurements from its parent to +// a single ResourceListener. +class BroadcastResourceListener::AdapterResource : public Resource { + public: + explicit AdapterResource(std::string name) : name_(std::move(name)) {} + ~AdapterResource() override { RTC_DCHECK(!listener_); } + + // The parent is letting us know we have a usage neasurement. + void OnResourceUsageStateMeasured(ResourceUsageState usage_state) { + rtc::CritScope crit(&lock_); + if (!listener_) + return; + listener_->OnResourceUsageStateMeasured(this, usage_state); + } + + // Resource implementation. + std::string Name() const override { return name_; } + void SetResourceListener(ResourceListener* listener) override { + rtc::CritScope crit(&lock_); + RTC_DCHECK(!listener_ || !listener); + listener_ = listener; + } + + private: + const std::string name_; + rtc::CriticalSection lock_; + ResourceListener* listener_ RTC_GUARDED_BY(lock_) = nullptr; +}; + +BroadcastResourceListener::BroadcastResourceListener( + rtc::scoped_refptr<Resource> source_resource) + : source_resource_(source_resource), is_listening_(false) { + RTC_DCHECK(source_resource_); +} + +BroadcastResourceListener::~BroadcastResourceListener() { + RTC_DCHECK(!is_listening_); +} + +rtc::scoped_refptr<Resource> BroadcastResourceListener::SourceResource() const { + return source_resource_; +} + +void BroadcastResourceListener::StartListening() { + rtc::CritScope crit(&lock_); + RTC_DCHECK(!is_listening_); + source_resource_->SetResourceListener(this); + is_listening_ = true; +} + +void BroadcastResourceListener::StopListening() { + rtc::CritScope crit(&lock_); + RTC_DCHECK(is_listening_); + RTC_DCHECK(adapters_.empty()); + source_resource_->SetResourceListener(nullptr); + is_listening_ = false; +} + +rtc::scoped_refptr<Resource> +BroadcastResourceListener::CreateAdapterResource() { + rtc::CritScope crit(&lock_); + RTC_DCHECK(is_listening_); + rtc::scoped_refptr<AdapterResource> adapter = + new rtc::RefCountedObject<AdapterResource>(source_resource_->Name() + + "Adapter"); + adapters_.push_back(adapter); + return adapter; +} + +void BroadcastResourceListener::RemoveAdapterResource( + rtc::scoped_refptr<Resource> resource) { + rtc::CritScope crit(&lock_); + auto it = std::find(adapters_.begin(), adapters_.end(), resource); + RTC_DCHECK(it != adapters_.end()); + adapters_.erase(it); +} + +std::vector<rtc::scoped_refptr<Resource>> +BroadcastResourceListener::GetAdapterResources() { + std::vector<rtc::scoped_refptr<Resource>> resources; + rtc::CritScope crit(&lock_); + for (const auto& adapter : adapters_) { + resources.push_back(adapter); + } + return resources; +} + +void BroadcastResourceListener::OnResourceUsageStateMeasured( + rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + RTC_DCHECK_EQ(resource, source_resource_); + rtc::CritScope crit(&lock_); + for (const auto& adapter : adapters_) { + adapter->OnResourceUsageStateMeasured(usage_state); + } +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener.h b/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener.h new file mode 100644 index 00000000000..f0d035dab7c --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener.h @@ -0,0 +1,75 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_BROADCAST_RESOURCE_LISTENER_H_ +#define CALL_ADAPTATION_BROADCAST_RESOURCE_LISTENER_H_ + +#include <vector> + +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" +#include "rtc_base/critical_section.h" + +namespace webrtc { + +// Responsible for forwarding 1 resource usage measurement to N listeners by +// creating N "adapter" resources. +// +// Example: +// If we have ResourceA, ResourceListenerX and ResourceListenerY we can create a +// BroadcastResourceListener that listens to ResourceA, use CreateAdapter() to +// spawn adapter resources ResourceX and ResourceY and let ResourceListenerX +// listen to ResourceX and ResourceListenerY listen to ResourceY. When ResourceA +// makes a measurement it will be echoed by both ResourceX and ResourceY. +// +// TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor is +// moved to call there will only be one ResourceAdaptationProcessor that needs +// to listen to the injected resources. When this is the case, delete this class +// and DCHECK that a Resource's listener is never overwritten. +class BroadcastResourceListener : public ResourceListener { + public: + explicit BroadcastResourceListener( + rtc::scoped_refptr<Resource> source_resource); + ~BroadcastResourceListener() override; + + rtc::scoped_refptr<Resource> SourceResource() const; + void StartListening(); + void StopListening(); + + // Creates a Resource that redirects any resource usage measurements that + // BroadcastResourceListener receives to its listener. + rtc::scoped_refptr<Resource> CreateAdapterResource(); + + // Unregister the adapter from the BroadcastResourceListener; it will no + // longer receive resource usage measurement and will no longer be referenced. + // Use this to prevent memory leaks of old adapters. + void RemoveAdapterResource(rtc::scoped_refptr<Resource> resource); + std::vector<rtc::scoped_refptr<Resource>> GetAdapterResources(); + + // ResourceListener implementation. + void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) override; + + private: + class AdapterResource; + friend class AdapterResource; + + const rtc::scoped_refptr<Resource> source_resource_; + rtc::CriticalSection lock_; + bool is_listening_ RTC_GUARDED_BY(lock_); + // The AdapterResource unregisters itself prior to destruction, guaranteeing + // that these pointers are safe to use. + std::vector<rtc::scoped_refptr<AdapterResource>> adapters_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_BROADCAST_RESOURCE_LISTENER_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener_unittest.cc b/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener_unittest.cc new file mode 100644 index 00000000000..9cd80500c2a --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/broadcast_resource_listener_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/broadcast_resource_listener.h" + +#include "call/adaptation/test/fake_resource.h" +#include "call/adaptation/test/mock_resource_listener.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +using ::testing::_; +using ::testing::StrictMock; + +TEST(BroadcastResourceListenerTest, CreateAndRemoveAdapterResource) { + rtc::scoped_refptr<FakeResource> source_resource = + FakeResource::Create("SourceResource"); + BroadcastResourceListener broadcast_resource_listener(source_resource); + broadcast_resource_listener.StartListening(); + + EXPECT_TRUE(broadcast_resource_listener.GetAdapterResources().empty()); + rtc::scoped_refptr<Resource> adapter = + broadcast_resource_listener.CreateAdapterResource(); + StrictMock<MockResourceListener> listener; + adapter->SetResourceListener(&listener); + EXPECT_EQ(std::vector<rtc::scoped_refptr<Resource>>{adapter}, + broadcast_resource_listener.GetAdapterResources()); + + // The removed adapter is not referenced by the broadcaster. + broadcast_resource_listener.RemoveAdapterResource(adapter); + EXPECT_TRUE(broadcast_resource_listener.GetAdapterResources().empty()); + // The removed adapter is not forwarding measurements. + EXPECT_CALL(listener, OnResourceUsageStateMeasured(_, _)).Times(0); + source_resource->SetUsageState(ResourceUsageState::kOveruse); + // Cleanup. + adapter->SetResourceListener(nullptr); + broadcast_resource_listener.StopListening(); +} + +TEST(BroadcastResourceListenerTest, AdapterNameIsBasedOnSourceResourceName) { + rtc::scoped_refptr<FakeResource> source_resource = + FakeResource::Create("FooBarResource"); + BroadcastResourceListener broadcast_resource_listener(source_resource); + broadcast_resource_listener.StartListening(); + + rtc::scoped_refptr<Resource> adapter = + broadcast_resource_listener.CreateAdapterResource(); + EXPECT_EQ("FooBarResourceAdapter", adapter->Name()); + + broadcast_resource_listener.RemoveAdapterResource(adapter); + broadcast_resource_listener.StopListening(); +} + +TEST(BroadcastResourceListenerTest, AdaptersForwardsUsageMeasurements) { + rtc::scoped_refptr<FakeResource> source_resource = + FakeResource::Create("SourceResource"); + BroadcastResourceListener broadcast_resource_listener(source_resource); + broadcast_resource_listener.StartListening(); + + StrictMock<MockResourceListener> destination_listener1; + StrictMock<MockResourceListener> destination_listener2; + rtc::scoped_refptr<Resource> adapter1 = + broadcast_resource_listener.CreateAdapterResource(); + adapter1->SetResourceListener(&destination_listener1); + rtc::scoped_refptr<Resource> adapter2 = + broadcast_resource_listener.CreateAdapterResource(); + adapter2->SetResourceListener(&destination_listener2); + + // Expect kOveruse to be echoed. + EXPECT_CALL(destination_listener1, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter1](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter1, resource); + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + EXPECT_CALL(destination_listener2, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter2](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter2, resource); + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + source_resource->SetUsageState(ResourceUsageState::kOveruse); + + // Expect kUnderuse to be echoed. + EXPECT_CALL(destination_listener1, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter1](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter1, resource); + EXPECT_EQ(ResourceUsageState::kUnderuse, usage_state); + }); + EXPECT_CALL(destination_listener2, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([adapter2](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(adapter2, resource); + EXPECT_EQ(ResourceUsageState::kUnderuse, usage_state); + }); + source_resource->SetUsageState(ResourceUsageState::kUnderuse); + + // Adapters have to be unregistered before they or the broadcaster is + // destroyed, ensuring safe use of raw pointers. + adapter1->SetResourceListener(nullptr); + adapter2->SetResourceListener(nullptr); + + broadcast_resource_listener.RemoveAdapterResource(adapter1); + broadcast_resource_listener.RemoveAdapterResource(adapter2); + broadcast_resource_listener.StopListening(); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/resource.cc b/chromium/third_party/webrtc/call/adaptation/resource.cc deleted file mode 100644 index a546450bc6b..00000000000 --- a/chromium/third_party/webrtc/call/adaptation/resource.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "call/adaptation/resource.h" - -#include "absl/algorithm/container.h" -#include "rtc_base/checks.h" - -namespace webrtc { - -ResourceListener::~ResourceListener() {} - -Resource::Resource() - : encoder_queue_(nullptr), - resource_adaptation_queue_(nullptr), - usage_state_(absl::nullopt), - listener_(nullptr) {} - -Resource::~Resource() { - RTC_DCHECK(!listener_) - << "There is a listener depending on a Resource being destroyed."; -} - -void Resource::Initialize(rtc::TaskQueue* encoder_queue, - rtc::TaskQueue* resource_adaptation_queue) { - RTC_DCHECK(!encoder_queue_); - RTC_DCHECK(encoder_queue); - RTC_DCHECK(!resource_adaptation_queue_); - RTC_DCHECK(resource_adaptation_queue); - encoder_queue_ = encoder_queue; - resource_adaptation_queue_ = resource_adaptation_queue; -} - -void Resource::SetResourceListener(ResourceListener* listener) { - RTC_DCHECK(resource_adaptation_queue_); - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - // If you want to change listener you need to unregister the old listener by - // setting it to null first. - RTC_DCHECK(!listener_ || !listener) << "A listener is already set"; - listener_ = listener; -} - -absl::optional<ResourceUsageState> Resource::usage_state() const { - RTC_DCHECK(resource_adaptation_queue_); - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - return usage_state_; -} - -void Resource::ClearUsageState() { - RTC_DCHECK(resource_adaptation_queue_); - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - usage_state_ = absl::nullopt; -} - -bool Resource::IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) const { - return true; -} - -void Resource::OnAdaptationApplied( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) {} - -rtc::TaskQueue* Resource::encoder_queue() const { - return encoder_queue_; -} - -rtc::TaskQueue* Resource::resource_adaptation_queue() const { - return resource_adaptation_queue_; -} - -void Resource::OnResourceUsageStateMeasured(ResourceUsageState usage_state) { - RTC_DCHECK(resource_adaptation_queue_); - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - usage_state_ = usage_state; - if (!listener_) - return; - listener_->OnResourceUsageStateMeasured(this); -} - -} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/resource.h b/chromium/third_party/webrtc/call/adaptation/resource.h deleted file mode 100644 index 2ee0c720d2c..00000000000 --- a/chromium/third_party/webrtc/call/adaptation/resource.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2019 The WebRTC Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef CALL_ADAPTATION_RESOURCE_H_ -#define CALL_ADAPTATION_RESOURCE_H_ - -#include <string> -#include <vector> - -#include "absl/types/optional.h" -#include "api/scoped_refptr.h" -#include "call/adaptation/video_source_restrictions.h" -#include "call/adaptation/video_stream_input_state.h" -#include "rtc_base/ref_count.h" -#include "rtc_base/task_queue.h" - -namespace webrtc { - -class Resource; - -enum class ResourceUsageState { - // Action is needed to minimze the load on this resource. - kOveruse, - // Increasing the load on this resource is desired, if possible. - kUnderuse, -}; - -class ResourceListener { - public: - virtual ~ResourceListener(); - - // Informs the listener of a new measurement of resource usage. This means - // that |resource->usage_state()| is now up-to-date. - virtual void OnResourceUsageStateMeasured( - rtc::scoped_refptr<Resource> resource) = 0; -}; - -class Resource : public rtc::RefCountInterface { - public: - // By default, usage_state() is null until a measurement is made. - Resource(); - ~Resource() override; - - void Initialize(rtc::TaskQueue* encoder_queue, - rtc::TaskQueue* resource_adaptation_queue); - - void SetResourceListener(ResourceListener* listener); - - absl::optional<ResourceUsageState> usage_state() const; - void ClearUsageState(); - - // This method allows the Resource to reject a proposed adaptation in the "up" - // direction if it predicts this would cause overuse of this resource. The - // default implementation unconditionally returns true (= allowed). - virtual bool IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) const; - virtual void OnAdaptationApplied( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource); - - virtual std::string name() const = 0; - - protected: - rtc::TaskQueue* encoder_queue() const; - rtc::TaskQueue* resource_adaptation_queue() const; - - // Updates the usage state and informs all registered listeners. - void OnResourceUsageStateMeasured(ResourceUsageState usage_state); - - private: - rtc::TaskQueue* encoder_queue_; - rtc::TaskQueue* resource_adaptation_queue_; - absl::optional<ResourceUsageState> usage_state_ - RTC_GUARDED_BY(resource_adaptation_queue_); - ResourceListener* listener_ RTC_GUARDED_BY(resource_adaptation_queue_); -}; - -} // namespace webrtc - -#endif // CALL_ADAPTATION_RESOURCE_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.cc b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.cc index 0224ac3bb2b..0be01cb7949 100644 --- a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.cc +++ b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.cc @@ -11,17 +11,66 @@ #include "call/adaptation/resource_adaptation_processor.h" #include <algorithm> +#include <string> #include <utility> #include "absl/algorithm/container.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/strings/string_builder.h" +#include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { +ResourceAdaptationProcessor::ResourceListenerDelegate::ResourceListenerDelegate( + ResourceAdaptationProcessor* processor) + : resource_adaptation_queue_(nullptr), processor_(processor) {} + +void ResourceAdaptationProcessor::ResourceListenerDelegate:: + SetResourceAdaptationQueue(TaskQueueBase* resource_adaptation_queue) { + RTC_DCHECK(!resource_adaptation_queue_); + RTC_DCHECK(resource_adaptation_queue); + resource_adaptation_queue_ = resource_adaptation_queue; + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); +} + +void ResourceAdaptationProcessor::ResourceListenerDelegate:: + OnProcessorDestroyed() { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + processor_ = nullptr; +} + +void ResourceAdaptationProcessor::ResourceListenerDelegate:: + OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + if (!resource_adaptation_queue_->IsCurrent()) { + resource_adaptation_queue_->PostTask(ToQueuedTask( + [this_ref = rtc::scoped_refptr<ResourceListenerDelegate>(this), + resource, usage_state] { + this_ref->OnResourceUsageStateMeasured(resource, usage_state); + })); + return; + } + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + if (processor_) { + processor_->OnResourceUsageStateMeasured(resource, usage_state); + } +} + +ResourceAdaptationProcessor::MitigationResultAndLogMessage:: + MitigationResultAndLogMessage() + : result(MitigationResult::kAdaptationApplied), message() {} + +ResourceAdaptationProcessor::MitigationResultAndLogMessage:: + MitigationResultAndLogMessage(MitigationResult result, std::string message) + : result(result), message(std::move(message)) {} + ResourceAdaptationProcessor::ResourceAdaptationProcessor( VideoStreamInputStateProvider* input_state_provider, VideoStreamEncoderObserver* encoder_stats_observer) - : sequence_checker_(), - is_resource_adaptation_enabled_(false), + : resource_adaptation_queue_(nullptr), + resource_listener_delegate_( + new rtc::RefCountedObject<ResourceListenerDelegate>(this)), input_state_provider_(input_state_provider), encoder_stats_observer_(encoder_stats_observer), resources_(), @@ -30,62 +79,123 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( is_screenshare_(false), stream_adapter_(std::make_unique<VideoStreamAdapter>()), last_reported_source_restrictions_(), - processing_in_progress_(false) { - sequence_checker_.Detach(); -} + previous_mitigation_results_(), + processing_in_progress_(false) {} ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DCHECK(!is_resource_adaptation_enabled_); - RTC_DCHECK(adaptation_listeners_.empty()) - << "There are listener(s) depending on a ResourceAdaptationProcessor " - << "being destroyed."; + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(restrictions_listeners_.empty()) + << "There are restrictions listener(s) depending on a " + << "ResourceAdaptationProcessor being destroyed."; RTC_DCHECK(resources_.empty()) << "There are resource(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; + RTC_DCHECK(adaptation_constraints_.empty()) + << "There are constaint(s) attached to a ResourceAdaptationProcessor " + << "being destroyed."; + RTC_DCHECK(adaptation_listeners_.empty()) + << "There are listener(s) attached to a ResourceAdaptationProcessor " + << "being destroyed."; + resource_listener_delegate_->OnProcessorDestroyed(); } -void ResourceAdaptationProcessor::InitializeOnResourceAdaptationQueue() { - // Allows |sequence_checker_| to attach to the resource adaptation queue. - // The caller is responsible for ensuring that this is the current queue. - RTC_DCHECK_RUN_ON(&sequence_checker_); +void ResourceAdaptationProcessor::SetResourceAdaptationQueue( + TaskQueueBase* resource_adaptation_queue) { + RTC_DCHECK(!resource_adaptation_queue_); + RTC_DCHECK(resource_adaptation_queue); + resource_adaptation_queue_ = resource_adaptation_queue; + resource_listener_delegate_->SetResourceAdaptationQueue( + resource_adaptation_queue); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); } DegradationPreference ResourceAdaptationProcessor::degradation_preference() const { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return degradation_preference_; } DegradationPreference ResourceAdaptationProcessor::effective_degradation_preference() const { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return effective_degradation_preference_; } -void ResourceAdaptationProcessor::StartResourceAdaptation() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - if (is_resource_adaptation_enabled_) - return; - for (const auto& resource : resources_) { - resource->SetResourceListener(this); - } - is_resource_adaptation_enabled_ = true; +void ResourceAdaptationProcessor::AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), + restrictions_listener) == restrictions_listeners_.end()); + restrictions_listeners_.push_back(restrictions_listener); } -void ResourceAdaptationProcessor::StopResourceAdaptation() { - RTC_DCHECK_RUN_ON(&sequence_checker_); - if (!is_resource_adaptation_enabled_) - return; - for (const auto& resource : resources_) { - resource->SetResourceListener(nullptr); +void ResourceAdaptationProcessor::RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + auto it = std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), restrictions_listener); + RTC_DCHECK(it != restrictions_listeners_.end()); + restrictions_listeners_.erase(it); +} + +void ResourceAdaptationProcessor::AddResource( + rtc::scoped_refptr<Resource> resource) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(resource); + RTC_DCHECK(absl::c_find(resources_, resource) == resources_.end()) + << "Resource \"" << resource->Name() << "\" was already registered."; + resources_.push_back(resource); + resource->SetResourceListener(resource_listener_delegate_); +} + +std::vector<rtc::scoped_refptr<Resource>> +ResourceAdaptationProcessor::GetResources() const { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + return resources_; +} + +void ResourceAdaptationProcessor::RemoveResource( + rtc::scoped_refptr<Resource> resource) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(resource); + RTC_LOG(INFO) << "Removing resource \"" << resource->Name() << "\"."; + auto it = absl::c_find(resources_, resource); + RTC_DCHECK(it != resources_.end()) << "Resource \"" << resource->Name() + << "\" was not a registered resource."; + auto resource_adaptation_limits = + adaptation_limits_by_resources_.find(resource); + if (resource_adaptation_limits != adaptation_limits_by_resources_.end()) { + VideoStreamAdapter::RestrictionsWithCounters adaptation_limits = + resource_adaptation_limits->second; + adaptation_limits_by_resources_.erase(resource_adaptation_limits); + MaybeUpdateResourceLimitationsOnResourceRemoval(adaptation_limits); } - is_resource_adaptation_enabled_ = false; + resources_.erase(it); + resource->SetResourceListener(nullptr); +} + +void ResourceAdaptationProcessor::AddAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), + adaptation_constraint) == adaptation_constraints_.end()); + adaptation_constraints_.push_back(adaptation_constraint); +} + +void ResourceAdaptationProcessor::RemoveAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + auto it = std::find(adaptation_constraints_.begin(), + adaptation_constraints_.end(), adaptation_constraint); + RTC_DCHECK(it != adaptation_constraints_.end()); + adaptation_constraints_.erase(it); } void ResourceAdaptationProcessor::AddAdaptationListener( - ResourceAdaptationProcessorListener* adaptation_listener) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + AdaptationListener* adaptation_listener) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(std::find(adaptation_listeners_.begin(), adaptation_listeners_.end(), adaptation_listener) == adaptation_listeners_.end()); @@ -93,52 +203,29 @@ void ResourceAdaptationProcessor::AddAdaptationListener( } void ResourceAdaptationProcessor::RemoveAdaptationListener( - ResourceAdaptationProcessorListener* adaptation_listener) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + AdaptationListener* adaptation_listener) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); auto it = std::find(adaptation_listeners_.begin(), adaptation_listeners_.end(), adaptation_listener); RTC_DCHECK(it != adaptation_listeners_.end()); adaptation_listeners_.erase(it); } -void ResourceAdaptationProcessor::AddResource( - rtc::scoped_refptr<Resource> resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); - // TODO(hbos): Allow adding resources while |is_resource_adaptation_enabled_| - // by registering as a listener of the resource on adding it. - RTC_DCHECK(!is_resource_adaptation_enabled_); - RTC_DCHECK(std::find(resources_.begin(), resources_.end(), resource) == - resources_.end()); - resources_.push_back(resource); -} - -void ResourceAdaptationProcessor::RemoveResource( - rtc::scoped_refptr<Resource> resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); - // TODO(hbos): Allow removing resources while - // |is_resource_adaptation_enabled_| by unregistering as a listener of the - // resource on removing it. - RTC_DCHECK(!is_resource_adaptation_enabled_); - auto it = std::find(resources_.begin(), resources_.end(), resource); - RTC_DCHECK(it != resources_.end()); - resources_.erase(it); -} - void ResourceAdaptationProcessor::SetDegradationPreference( DegradationPreference degradation_preference) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); degradation_preference_ = degradation_preference; MaybeUpdateEffectiveDegradationPreference(); } void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); is_screenshare_ = is_screenshare; MaybeUpdateEffectiveDegradationPreference(); } void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); effective_degradation_preference_ = (is_screenshare_ && degradation_preference_ == DegradationPreference::BALANCED) @@ -149,76 +236,103 @@ void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { } void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_LOG(INFO) << "Resetting restrictions"; stream_adapter_->ClearRestrictions(); - adaptations_counts_by_resource_.clear(); + adaptation_limits_by_resources_.clear(); + for (auto restrictions_listener : restrictions_listeners_) { + restrictions_listener->OnResourceLimitationChanged(nullptr, {}); + } MaybeUpdateVideoSourceRestrictions(nullptr); } void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( rtc::scoped_refptr<Resource> reason) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); VideoSourceRestrictions new_source_restrictions = FilterRestrictionsByDegradationPreference( stream_adapter_->source_restrictions(), effective_degradation_preference_); if (last_reported_source_restrictions_ != new_source_restrictions) { + RTC_LOG(INFO) << "Reporting new restrictions (in " + << DegradationPreferenceToString( + effective_degradation_preference_) + << "): " << new_source_restrictions.ToString(); last_reported_source_restrictions_ = std::move(new_source_restrictions); - for (auto* adaptation_listener : adaptation_listeners_) { - adaptation_listener->OnVideoSourceRestrictionsUpdated( + for (auto* restrictions_listener : restrictions_listeners_) { + restrictions_listener->OnVideoSourceRestrictionsUpdated( last_reported_source_restrictions_, stream_adapter_->adaptation_counters(), reason); } - if (reason) { - UpdateResourceDegradationCounts(reason); - } } } void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( - rtc::scoped_refptr<Resource> resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DCHECK(resource->usage_state().has_value()); - switch (resource->usage_state().value()) { + rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_DCHECK(resource); + // |resource| could have been removed after signalling. + if (absl::c_find(resources_, resource) == resources_.end()) { + RTC_LOG(INFO) << "Ignoring signal from removed resource \"" + << resource->Name() << "\"."; + return; + } + MitigationResultAndLogMessage result_and_message; + switch (usage_state) { case ResourceUsageState::kOveruse: - OnResourceOveruse(resource); + result_and_message = OnResourceOveruse(resource); break; case ResourceUsageState::kUnderuse: - OnResourceUnderuse(resource); + result_and_message = OnResourceUnderuse(resource); break; } + // Maybe log the result of the operation. + auto it = previous_mitigation_results_.find(resource.get()); + if (it != previous_mitigation_results_.end() && + it->second == result_and_message.result) { + // This resource has previously reported the same result and we haven't + // successfully adapted since - don't log to avoid spam. + return; + } + RTC_LOG(INFO) << "Resource \"" << resource->Name() << "\" signalled " + << ResourceUsageStateToString(usage_state) << ". " + << result_and_message.message; + if (result_and_message.result == MitigationResult::kAdaptationApplied) { + previous_mitigation_results_.clear(); + } else { + previous_mitigation_results_.insert( + std::make_pair(resource.get(), result_and_message.result)); + } } bool ResourceAdaptationProcessor::HasSufficientInputForAdaptation( const VideoStreamInputState& input_state) const { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); return input_state.HasInputFrameSizeAndFramesPerSecond() && (effective_degradation_preference_ != DegradationPreference::MAINTAIN_RESOLUTION || input_state.frames_per_second() >= kMinFrameRateFps); } -void ResourceAdaptationProcessor::OnResourceUnderuse( +ResourceAdaptationProcessor::MitigationResultAndLogMessage +ResourceAdaptationProcessor::OnResourceUnderuse( rtc::scoped_refptr<Resource> reason_resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; - // Clear all usage states. In order to re-run adaptation logic, resources need - // to provide new resource usage measurements. - // TODO(hbos): Support not unconditionally clearing usage states by having the - // ResourceAdaptationProcessor check in on its resources at certain intervals. - for (const auto& resource : resources_) { - resource->ClearUsageState(); - } - VideoStreamInputState input_state = input_state_provider_->InputState(); - if (effective_degradation_preference_ == DegradationPreference::DISABLED || - !HasSufficientInputForAdaptation(input_state)) { + if (effective_degradation_preference_ == DegradationPreference::DISABLED) { processing_in_progress_ = false; - return; + return MitigationResultAndLogMessage( + MitigationResult::kDisabled, + "Not adapting up because DegradationPreference is disabled"); } - if (!IsResourceAllowedToAdaptUp(reason_resource)) { + VideoStreamInputState input_state = input_state_provider_->InputState(); + if (!HasSufficientInputForAdaptation(input_state)) { processing_in_progress_ = false; - return; + return MitigationResultAndLogMessage( + MitigationResult::kInsufficientInput, + "Not adapting up because input is insufficient"); } // Update video input states and encoder settings for accurate adaptation. stream_adapter_->SetInput(input_state); @@ -226,56 +340,99 @@ void ResourceAdaptationProcessor::OnResourceUnderuse( Adaptation adaptation = stream_adapter_->GetAdaptationUp(); if (adaptation.status() != Adaptation::Status::kValid) { processing_in_progress_ = false; - return; + rtc::StringBuilder message; + message << "Not adapting up because VideoStreamAdapter returned " + << Adaptation::StatusToString(adaptation.status()); + return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, + message.Release()); } - // Are all resources OK with this adaptation being applied? VideoSourceRestrictions restrictions_before = stream_adapter_->source_restrictions(); - VideoSourceRestrictions restrictions_after = + VideoStreamAdapter::RestrictionsWithCounters peek_restrictions = stream_adapter_->PeekNextRestrictions(adaptation); - if (!absl::c_all_of(resources_, [&input_state, &restrictions_before, - &restrictions_after, &reason_resource]( - rtc::scoped_refptr<Resource> resource) { - return resource->IsAdaptationUpAllowed(input_state, restrictions_before, - restrictions_after, - reason_resource); - })) { - processing_in_progress_ = false; - return; + VideoSourceRestrictions restrictions_after = peek_restrictions.restrictions; + // Check that resource is most limited... + std::vector<rtc::scoped_refptr<Resource>> most_limited_resources; + VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions; + std::tie(most_limited_resources, most_limited_restrictions) = + FindMostLimitedResources(); + + for (const auto* constraint : adaptation_constraints_) { + if (!constraint->IsAdaptationUpAllowed(input_state, restrictions_before, + restrictions_after, + reason_resource)) { + processing_in_progress_ = false; + rtc::StringBuilder message; + message << "Not adapting up because constraint \"" << constraint->Name() + << "\" disallowed it"; + return MitigationResultAndLogMessage( + MitigationResult::kRejectedByConstraint, message.Release()); + } + } + // If the most restricted resource is less limited than current restrictions + // then proceed with adapting up. + if (!most_limited_resources.empty() && + most_limited_restrictions.adaptation_counters.Total() >= + stream_adapter_->adaptation_counters().Total()) { + // If |reason_resource| is not one of the most limiting resources then abort + // adaptation. + if (absl::c_find(most_limited_resources, reason_resource) == + most_limited_resources.end()) { + processing_in_progress_ = false; + rtc::StringBuilder message; + message << "Resource \"" << reason_resource->Name() + << "\" was not the most limited resource."; + return MitigationResultAndLogMessage( + MitigationResult::kNotMostLimitedResource, message.Release()); + } + + UpdateResourceLimitations(reason_resource, peek_restrictions); + if (most_limited_resources.size() > 1) { + // If there are multiple most limited resources, all must signal underuse + // before the adaptation is applied. + processing_in_progress_ = false; + rtc::StringBuilder message; + message << "Resource \"" << reason_resource->Name() + << "\" was not the only most limited resource."; + return MitigationResultAndLogMessage( + MitigationResult::kSharedMostLimitedResource, message.Release()); + } } // Apply adaptation. stream_adapter_->ApplyAdaptation(adaptation); - for (const auto& resource : resources_) { - resource->OnAdaptationApplied(input_state, restrictions_before, - restrictions_after, reason_resource); + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnAdaptationApplied( + input_state, restrictions_before, restrictions_after, reason_resource); } // Update VideoSourceRestrictions based on adaptation. This also informs the - // |adaptation_listeners_|. + // |restrictions_listeners_|. MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; + rtc::StringBuilder message; + message << "Adapted up successfully. Unfiltered adaptations: " + << stream_adapter_->adaptation_counters().ToString(); + return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied, + message.Release()); } -void ResourceAdaptationProcessor::OnResourceOveruse( +ResourceAdaptationProcessor::MitigationResultAndLogMessage +ResourceAdaptationProcessor::OnResourceOveruse( rtc::scoped_refptr<Resource> reason_resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); RTC_DCHECK(!processing_in_progress_); processing_in_progress_ = true; - // Clear all usage states. In order to re-run adaptation logic, resources need - // to provide new resource usage measurements. - // TODO(hbos): Support not unconditionally clearing usage states by having the - // ResourceAdaptationProcessor check in on its resources at certain intervals. - for (const auto& resource : resources_) { - resource->ClearUsageState(); + if (effective_degradation_preference_ == DegradationPreference::DISABLED) { + processing_in_progress_ = false; + return MitigationResultAndLogMessage( + MitigationResult::kDisabled, + "Not adapting down because DegradationPreference is disabled"); } VideoStreamInputState input_state = input_state_provider_->InputState(); - if (!input_state.has_input()) { + if (!HasSufficientInputForAdaptation(input_state)) { processing_in_progress_ = false; - return; - } - if (effective_degradation_preference_ == DegradationPreference::DISABLED || - !HasSufficientInputForAdaptation(input_state)) { - processing_in_progress_ = false; - return; + return MitigationResultAndLogMessage( + MitigationResult::kInsufficientInput, + "Not adapting down because input is insufficient"); } // Update video input states and encoder settings for accurate adaptation. stream_adapter_->SetInput(input_state); @@ -286,27 +443,40 @@ void ResourceAdaptationProcessor::OnResourceOveruse( } if (adaptation.status() != Adaptation::Status::kValid) { processing_in_progress_ = false; - return; + rtc::StringBuilder message; + message << "Not adapting down because VideoStreamAdapter returned " + << Adaptation::StatusToString(adaptation.status()); + return MitigationResultAndLogMessage(MitigationResult::kRejectedByAdapter, + message.Release()); } // Apply adaptation. VideoSourceRestrictions restrictions_before = stream_adapter_->source_restrictions(); - VideoSourceRestrictions restrictions_after = + VideoStreamAdapter::RestrictionsWithCounters peek_next_restrictions = stream_adapter_->PeekNextRestrictions(adaptation); + VideoSourceRestrictions restrictions_after = + peek_next_restrictions.restrictions; + UpdateResourceLimitations(reason_resource, peek_next_restrictions); stream_adapter_->ApplyAdaptation(adaptation); - for (const auto& resource : resources_) { - resource->OnAdaptationApplied(input_state, restrictions_before, - restrictions_after, reason_resource); + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnAdaptationApplied( + input_state, restrictions_before, restrictions_after, reason_resource); } // Update VideoSourceRestrictions based on adaptation. This also informs the - // |adaptation_listeners_|. + // |restrictions_listeners_|. MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; + rtc::StringBuilder message; + message << "Adapted down successfully. Unfiltered adaptations: " + << stream_adapter_->adaptation_counters().ToString(); + return MitigationResultAndLogMessage(MitigationResult::kAdaptationApplied, + message.Release()); } void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( rtc::scoped_refptr<Resource> reason_resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + RTC_LOG(INFO) << "TriggerAdaptationDueToFrameDroppedDueToSize called"; VideoAdaptationCounters counters_before = stream_adapter_->adaptation_counters(); OnResourceOveruse(reason_resource); @@ -323,27 +493,85 @@ void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize( } } -void ResourceAdaptationProcessor::UpdateResourceDegradationCounts( - rtc::scoped_refptr<Resource> resource) { - RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DCHECK(resource); - int delta = stream_adapter_->adaptation_counters().Total(); - for (const auto& adaptations : adaptations_counts_by_resource_) { - delta -= adaptations.second; +std::pair<std::vector<rtc::scoped_refptr<Resource>>, + VideoStreamAdapter::RestrictionsWithCounters> +ResourceAdaptationProcessor::FindMostLimitedResources() const { + std::vector<rtc::scoped_refptr<Resource>> most_limited_resources; + VideoStreamAdapter::RestrictionsWithCounters most_limited_restrictions{ + VideoSourceRestrictions(), VideoAdaptationCounters()}; + + for (const auto& resource_and_adaptation_limit_ : + adaptation_limits_by_resources_) { + const auto& restrictions_with_counters = + resource_and_adaptation_limit_.second; + if (restrictions_with_counters.adaptation_counters.Total() > + most_limited_restrictions.adaptation_counters.Total()) { + most_limited_restrictions = restrictions_with_counters; + most_limited_resources.clear(); + most_limited_resources.push_back(resource_and_adaptation_limit_.first); + } else if (most_limited_restrictions.adaptation_counters == + restrictions_with_counters.adaptation_counters) { + most_limited_resources.push_back(resource_and_adaptation_limit_.first); + } + } + return std::make_pair(std::move(most_limited_resources), + most_limited_restrictions); +} + +void ResourceAdaptationProcessor::UpdateResourceLimitations( + rtc::scoped_refptr<Resource> reason_resource, + const VideoStreamAdapter::RestrictionsWithCounters& + peek_next_restrictions) { + adaptation_limits_by_resources_[reason_resource] = peek_next_restrictions; + + std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters> limitations; + for (const auto& p : adaptation_limits_by_resources_) { + limitations.insert(std::make_pair(p.first, p.second.adaptation_counters)); } - // Default value is 0, inserts the value if missing. - adaptations_counts_by_resource_[resource] += delta; - RTC_DCHECK_GE(adaptations_counts_by_resource_[resource], 0); + for (auto restrictions_listener : restrictions_listeners_) { + restrictions_listener->OnResourceLimitationChanged(reason_resource, + limitations); + } } -bool ResourceAdaptationProcessor::IsResourceAllowedToAdaptUp( - rtc::scoped_refptr<Resource> resource) const { - RTC_DCHECK_RUN_ON(&sequence_checker_); - RTC_DCHECK(resource); - const auto& adaptations = adaptations_counts_by_resource_.find(resource); - return adaptations != adaptations_counts_by_resource_.end() && - adaptations->second > 0; +void ResourceAdaptationProcessor:: + MaybeUpdateResourceLimitationsOnResourceRemoval( + VideoStreamAdapter::RestrictionsWithCounters removed_limitations) { + if (adaptation_limits_by_resources_.empty()) { + // Only the resource being removed was adapted so reset restrictions. + ResetVideoSourceRestrictions(); + return; + } + + VideoStreamAdapter::RestrictionsWithCounters most_limited = + FindMostLimitedResources().second; + + if (removed_limitations.adaptation_counters.Total() <= + most_limited.adaptation_counters.Total()) { + // The removed limitations were less limited than the most limited resource. + // Don't change the current restrictions. + return; + } + + // Apply the new most limited resource as the next restrictions. + Adaptation adapt_to = stream_adapter_->GetAdaptationTo( + most_limited.adaptation_counters, most_limited.restrictions); + RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid); + stream_adapter_->ApplyAdaptation(adapt_to); + + RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to " + "next most limited restrictions: " + << most_limited.restrictions.ToString() << " with counters " + << most_limited.adaptation_counters.ToString(); + + MaybeUpdateVideoSourceRestrictions(nullptr); + auto input_state = input_state_provider_->InputState(); + for (auto* adaptation_listener : adaptation_listeners_) { + adaptation_listener->OnAdaptationApplied( + input_state, removed_limitations.restrictions, + most_limited.restrictions, nullptr); + } } } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.h b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.h index cf1e187026a..cff50955e70 100644 --- a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.h +++ b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor.h @@ -13,20 +13,24 @@ #include <map> #include <memory> +#include <string> +#include <utility> #include <vector> #include "absl/types/optional.h" +#include "api/adaptation/resource.h" #include "api/rtp_parameters.h" #include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_base.h" #include "api/video/video_frame.h" #include "api/video/video_stream_encoder_observer.h" -#include "call/adaptation/resource.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/adaptation_listener.h" #include "call/adaptation/resource_adaptation_processor_interface.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_adapter.h" #include "call/adaptation/video_stream_input_state.h" #include "call/adaptation/video_stream_input_state_provider.h" -#include "rtc_base/synchronization/sequence_checker.h" namespace webrtc { @@ -54,20 +58,27 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, VideoStreamEncoderObserver* encoder_stats_observer); ~ResourceAdaptationProcessor() override; - void InitializeOnResourceAdaptationQueue() override; + void SetResourceAdaptationQueue( + TaskQueueBase* resource_adaptation_queue) override; // ResourceAdaptationProcessorInterface implementation. DegradationPreference degradation_preference() const override; DegradationPreference effective_degradation_preference() const override; - void StartResourceAdaptation() override; - void StopResourceAdaptation() override; - void AddAdaptationListener( - ResourceAdaptationProcessorListener* adaptation_listener) override; - void RemoveAdaptationListener( - ResourceAdaptationProcessorListener* adaptation_listener) override; + void AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) override; + void RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) override; void AddResource(rtc::scoped_refptr<Resource> resource) override; + std::vector<rtc::scoped_refptr<Resource>> GetResources() const override; void RemoveResource(rtc::scoped_refptr<Resource> resource) override; + void AddAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) override; + void RemoveAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) override; + void AddAdaptationListener(AdaptationListener* adaptation_listener) override; + void RemoveAdaptationListener( + AdaptationListener* adaptation_listener) override; void SetDegradationPreference( DegradationPreference degradation_preference) override; @@ -76,8 +87,8 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // ResourceListener implementation. // Triggers OnResourceUnderuse() or OnResourceOveruse(). - void OnResourceUsageStateMeasured( - rtc::scoped_refptr<Resource> resource) override; + void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) override; // May trigger 1-2 adaptations. It is meant to reduce resolution but this is // not guaranteed. It may adapt frame rate, which does not address the issue. @@ -89,11 +100,51 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, bool HasSufficientInputForAdaptation( const VideoStreamInputState& input_state) const; + // If resource usage measurements happens off the adaptation task queue, this + // class takes care of posting the measurement for the processor to handle it + // on the adaptation task queue. + class ResourceListenerDelegate : public rtc::RefCountInterface, + public ResourceListener { + public: + explicit ResourceListenerDelegate(ResourceAdaptationProcessor* processor); + + void SetResourceAdaptationQueue(TaskQueueBase* resource_adaptation_queue); + void OnProcessorDestroyed(); + + // ResourceListener implementation. + void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) override; + + private: + TaskQueueBase* resource_adaptation_queue_; + ResourceAdaptationProcessor* processor_ + RTC_GUARDED_BY(resource_adaptation_queue_); + }; + + enum class MitigationResult { + kDisabled, + kInsufficientInput, + kNotMostLimitedResource, + kSharedMostLimitedResource, + kRejectedByAdapter, + kRejectedByConstraint, + kAdaptationApplied, + }; + + struct MitigationResultAndLogMessage { + MitigationResultAndLogMessage(); + MitigationResultAndLogMessage(MitigationResult result, std::string message); + MitigationResult result; + std::string message; + }; + // Performs the adaptation by getting the next target, applying it and // informing listeners of the new VideoSourceRestriction and adaptation // counters. - void OnResourceUnderuse(rtc::scoped_refptr<Resource> reason_resource); - void OnResourceOveruse(rtc::scoped_refptr<Resource> reason_resource); + MitigationResultAndLogMessage OnResourceUnderuse( + rtc::scoped_refptr<Resource> reason_resource); + MitigationResultAndLogMessage OnResourceOveruse( + rtc::scoped_refptr<Resource> reason_resource); // Needs to be invoked any time |degradation_preference_| or |is_screenshare_| // changes to ensure |effective_degradation_preference_| is up-to-date. @@ -101,43 +152,60 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // If the filtered source restrictions are different than // |last_reported_source_restrictions_|, inform the listeners. void MaybeUpdateVideoSourceRestrictions(rtc::scoped_refptr<Resource> reason); - // Updates the number of times the resource has degraded based on the latest - // degradation applied. - void UpdateResourceDegradationCounts(rtc::scoped_refptr<Resource> resource); - // Returns true if a Resource has been overused in the pass and is responsible - // for creating a VideoSourceRestriction. The current algorithm counts the - // number of times the resource caused an adaptation and allows adapting up - // if that number is non-zero. This is consistent with how adaptation has - // traditionally been handled. - // TODO(crbug.com/webrtc/11553) Change this algorithm to look at the resources - // restrictions rather than just the counters. - bool IsResourceAllowedToAdaptUp(rtc::scoped_refptr<Resource> resource) const; - - webrtc::SequenceChecker sequence_checker_; - bool is_resource_adaptation_enabled_ RTC_GUARDED_BY(sequence_checker_); + + void UpdateResourceLimitations( + rtc::scoped_refptr<Resource> reason_resource, + const VideoStreamAdapter::RestrictionsWithCounters& + peek_next_restrictions) RTC_RUN_ON(resource_adaptation_queue_); + + // Searches |adaptation_limits_by_resources_| for each resource with the + // highest total adaptation counts. Adaptation up may only occur if the + // resource performing the adaptation is the only most limited resource. This + // function returns the list of all most limited resources as well as the + // corresponding adaptation of that resource. + std::pair<std::vector<rtc::scoped_refptr<Resource>>, + VideoStreamAdapter::RestrictionsWithCounters> + FindMostLimitedResources() const RTC_RUN_ON(resource_adaptation_queue_); + + void MaybeUpdateResourceLimitationsOnResourceRemoval( + VideoStreamAdapter::RestrictionsWithCounters removed_limitations) + RTC_RUN_ON(resource_adaptation_queue_); + + TaskQueueBase* resource_adaptation_queue_; + rtc::scoped_refptr<ResourceListenerDelegate> resource_listener_delegate_; // Input and output. VideoStreamInputStateProvider* const input_state_provider_ - RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); VideoStreamEncoderObserver* const encoder_stats_observer_ - RTC_GUARDED_BY(sequence_checker_); - std::vector<ResourceAdaptationProcessorListener*> adaptation_listeners_ - RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); + std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_ + RTC_GUARDED_BY(resource_adaptation_queue_); std::vector<rtc::scoped_refptr<Resource>> resources_ - RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); + std::vector<AdaptationConstraint*> adaptation_constraints_ + RTC_GUARDED_BY(resource_adaptation_queue_); + std::vector<AdaptationListener*> adaptation_listeners_ + RTC_GUARDED_BY(resource_adaptation_queue_); // Purely used for statistics, does not ensure mapped resources stay alive. - std::map<const Resource*, int> adaptations_counts_by_resource_ - RTC_GUARDED_BY(sequence_checker_); + std::map<rtc::scoped_refptr<Resource>, + VideoStreamAdapter::RestrictionsWithCounters> + adaptation_limits_by_resources_ + RTC_GUARDED_BY(resource_adaptation_queue_); // Adaptation strategy settings. DegradationPreference degradation_preference_ - RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); DegradationPreference effective_degradation_preference_ - RTC_GUARDED_BY(sequence_checker_); - bool is_screenshare_ RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); + bool is_screenshare_ RTC_GUARDED_BY(resource_adaptation_queue_); // Responsible for generating and applying possible adaptations. const std::unique_ptr<VideoStreamAdapter> stream_adapter_ - RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); VideoSourceRestrictions last_reported_source_restrictions_ - RTC_GUARDED_BY(sequence_checker_); + RTC_GUARDED_BY(resource_adaptation_queue_); + // Keeps track of previous mitigation results per resource since the last + // successful adaptation. Used to avoid RTC_LOG spam. + std::map<Resource*, MitigationResult> previous_mitigation_results_ + RTC_GUARDED_BY(resource_adaptation_queue_); // Prevents recursion. // // This is used to prevent triggering resource adaptation in the process of @@ -149,7 +217,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // Resource::OnAdaptationApplied() -> // Resource::OnResourceUsageStateMeasured() -> // ResourceAdaptationProcessor::OnResourceOveruse() // Boom, not allowed. - bool processing_in_progress_ RTC_GUARDED_BY(sequence_checker_); + bool processing_in_progress_ RTC_GUARDED_BY(resource_adaptation_queue_); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.cc b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.cc index 4e5251ce909..48ddf65ed3b 100644 --- a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.cc +++ b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.cc @@ -12,8 +12,9 @@ namespace webrtc { -ResourceAdaptationProcessorListener::~ResourceAdaptationProcessorListener() {} +VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default; -ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() {} +ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() = + default; } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.h b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.h index d6295c4d750..a97fe8efe4b 100644 --- a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.h +++ b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_interface.h @@ -11,23 +11,28 @@ #ifndef CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ #define CALL_ADAPTATION_RESOURCE_ADAPTATION_PROCESSOR_INTERFACE_H_ +#include <map> +#include <vector> + #include "absl/types/optional.h" +#include "api/adaptation/resource.h" #include "api/rtp_parameters.h" #include "api/scoped_refptr.h" +#include "api/task_queue/task_queue_base.h" #include "api/video/video_adaptation_counters.h" #include "api/video/video_frame.h" +#include "call/adaptation/adaptation_constraint.h" +#include "call/adaptation/adaptation_listener.h" #include "call/adaptation/encoder_settings.h" -#include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" -#include "rtc_base/task_queue.h" namespace webrtc { // The listener is responsible for carrying out the reconfiguration of the video // source such that the VideoSourceRestrictions are fulfilled. -class ResourceAdaptationProcessorListener { +class VideoSourceRestrictionsListener { public: - virtual ~ResourceAdaptationProcessorListener(); + virtual ~VideoSourceRestrictionsListener(); // The |restrictions| are filtered by degradation preference but not the // |adaptation_counters|, which are currently only reported for legacy stats @@ -36,6 +41,13 @@ class ResourceAdaptationProcessorListener { VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, rtc::scoped_refptr<Resource> reason) = 0; + + // The limitations on a resource were changed. This does not mean the current + // video restrictions have changed. + virtual void OnResourceLimitationChanged( + rtc::scoped_refptr<Resource> resource, + const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>& + resource_limitations) {} }; // The Resource Adaptation Processor is responsible for reacting to resource @@ -46,7 +58,8 @@ class ResourceAdaptationProcessorInterface { public: virtual ~ResourceAdaptationProcessorInterface(); - virtual void InitializeOnResourceAdaptationQueue() = 0; + virtual void SetResourceAdaptationQueue( + TaskQueueBase* resource_adaptation_queue) = 0; virtual DegradationPreference degradation_preference() const = 0; // Reinterprets "balanced + screenshare" as "maintain-resolution". @@ -61,14 +74,21 @@ class ResourceAdaptationProcessorInterface { // with AddResource() and RemoveResource() instead. When the processor is // multi-stream aware, stream-specific resouces will get added and removed // over time. - virtual void StartResourceAdaptation() = 0; - virtual void StopResourceAdaptation() = 0; - virtual void AddAdaptationListener( - ResourceAdaptationProcessorListener* adaptation_listener) = 0; - virtual void RemoveAdaptationListener( - ResourceAdaptationProcessorListener* adaptation_listener) = 0; + virtual void AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) = 0; + virtual void RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) = 0; virtual void AddResource(rtc::scoped_refptr<Resource> resource) = 0; + virtual std::vector<rtc::scoped_refptr<Resource>> GetResources() const = 0; virtual void RemoveResource(rtc::scoped_refptr<Resource> resource) = 0; + virtual void AddAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) = 0; + virtual void RemoveAdaptationConstraint( + AdaptationConstraint* adaptation_constraint) = 0; + virtual void AddAdaptationListener( + AdaptationListener* adaptation_listener) = 0; + virtual void RemoveAdaptationListener( + AdaptationListener* adaptation_listener) = 0; virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; diff --git a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_unittest.cc b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_unittest.cc index e94b3a99d7e..da1ab1cda1d 100644 --- a/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/chromium/third_party/webrtc/call/adaptation/resource_adaptation_processor_unittest.cc @@ -10,15 +10,19 @@ #include "call/adaptation/resource_adaptation_processor.h" +#include "api/adaptation/resource.h" #include "api/scoped_refptr.h" #include "api/video/video_adaptation_counters.h" -#include "call/adaptation/resource.h" #include "call/adaptation/resource_adaptation_processor_interface.h" +#include "call/adaptation/test/fake_adaptation_constraint.h" +#include "call/adaptation/test/fake_adaptation_listener.h" #include "call/adaptation/test/fake_frame_rate_provider.h" #include "call/adaptation/test/fake_resource.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state_provider.h" +#include "rtc_base/critical_section.h" #include "rtc_base/event.h" +#include "rtc_base/gunit.h" #include "rtc_base/task_queue_for_test.h" #include "test/gtest.h" @@ -28,31 +32,41 @@ namespace { const int kDefaultFrameRate = 30; const int kDefaultFrameSize = 1280 * 720; +const int kDefaultTimeoutMs = 5000; -class ResourceAdaptationProcessorListenerForTesting - : public ResourceAdaptationProcessorListener { +class VideoSourceRestrictionsListenerForTesting + : public VideoSourceRestrictionsListener { public: - ResourceAdaptationProcessorListenerForTesting() + VideoSourceRestrictionsListenerForTesting() : restrictions_updated_count_(0), restrictions_(), adaptation_counters_(), reason_(nullptr) {} - ~ResourceAdaptationProcessorListenerForTesting() override {} + ~VideoSourceRestrictionsListenerForTesting() override {} size_t restrictions_updated_count() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return restrictions_updated_count_; } - const VideoSourceRestrictions& restrictions() const { return restrictions_; } - const VideoAdaptationCounters& adaptation_counters() const { + VideoSourceRestrictions restrictions() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return restrictions_; + } + VideoAdaptationCounters adaptation_counters() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return adaptation_counters_; } - rtc::scoped_refptr<Resource> reason() const { return reason_; } + rtc::scoped_refptr<Resource> reason() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return reason_; + } - // ResourceAdaptationProcessorListener implementation. + // VideoSourceRestrictionsListener implementation. void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, rtc::scoped_refptr<Resource> reason) override { + RTC_DCHECK_RUN_ON(&sequence_checker_); ++restrictions_updated_count_; restrictions_ = restrictions; adaptation_counters_ = adaptation_counters; @@ -60,47 +74,37 @@ class ResourceAdaptationProcessorListenerForTesting } private: - size_t restrictions_updated_count_; - VideoSourceRestrictions restrictions_; - VideoAdaptationCounters adaptation_counters_; - rtc::scoped_refptr<Resource> reason_; + SequenceChecker sequence_checker_; + size_t restrictions_updated_count_ RTC_GUARDED_BY(&sequence_checker_); + VideoSourceRestrictions restrictions_ RTC_GUARDED_BY(&sequence_checker_); + VideoAdaptationCounters adaptation_counters_ + RTC_GUARDED_BY(&sequence_checker_); + rtc::scoped_refptr<Resource> reason_ RTC_GUARDED_BY(&sequence_checker_); }; class ResourceAdaptationProcessorTest : public ::testing::Test { public: ResourceAdaptationProcessorTest() - : resource_adaptation_queue_("ResourceAdaptationQueue"), - encoder_queue_("EncoderQueue"), - frame_rate_provider_(), + : frame_rate_provider_(), input_state_provider_(&frame_rate_provider_), - resource_(new FakeResource("FakeResource")), - other_resource_(new FakeResource("OtherFakeResource")), + resource_(FakeResource::Create("FakeResource")), + other_resource_(FakeResource::Create("OtherFakeResource")), + adaptation_constraint_("FakeAdaptationConstraint"), + adaptation_listener_(), processor_(std::make_unique<ResourceAdaptationProcessor>( &input_state_provider_, /*encoder_stats_observer=*/&frame_rate_provider_)) { - resource_->Initialize(&encoder_queue_, &resource_adaptation_queue_); - other_resource_->Initialize(&encoder_queue_, &resource_adaptation_queue_); - rtc::Event event; - resource_adaptation_queue_.PostTask([this, &event] { - processor_->InitializeOnResourceAdaptationQueue(); - processor_->AddAdaptationListener(&processor_listener_); - processor_->AddResource(resource_); - processor_->AddResource(other_resource_); - event.Set(); - }); - event.Wait(rtc::Event::kForever); + processor_->SetResourceAdaptationQueue(TaskQueueBase::Current()); + processor_->AddRestrictionsListener(&restrictions_listener_); + processor_->AddResource(resource_); + processor_->AddResource(other_resource_); + processor_->AddAdaptationConstraint(&adaptation_constraint_); + processor_->AddAdaptationListener(&adaptation_listener_); } ~ResourceAdaptationProcessorTest() override { - rtc::Event event; - resource_adaptation_queue_.PostTask([this, &event] { - processor_->StopResourceAdaptation(); - processor_->RemoveResource(resource_); - processor_->RemoveResource(other_resource_); - processor_->RemoveAdaptationListener(&processor_listener_); - processor_.reset(); - event.Set(); - }); - event.Wait(rtc::Event::kForever); + if (processor_) { + DestroyProcessor(); + } } void SetInputStates(bool has_input, int fps, int frame_size) { @@ -117,53 +121,60 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { : restrictions.max_pixels_per_frame().value_or(kDefaultFrameSize)); } + void DestroyProcessor() { + processor_->RemoveRestrictionsListener(&restrictions_listener_); + if (resource_) { + processor_->RemoveResource(resource_); + } + if (other_resource_) { + processor_->RemoveResource(other_resource_); + } + processor_->RemoveAdaptationConstraint(&adaptation_constraint_); + processor_->RemoveAdaptationListener(&adaptation_listener_); + processor_.reset(); + } + + static void WaitUntilTaskQueueIdle() { + ASSERT_TRUE(rtc::Thread::Current()->ProcessMessages(0)); + } + protected: - TaskQueueForTest resource_adaptation_queue_; - TaskQueueForTest encoder_queue_; FakeFrameRateProvider frame_rate_provider_; VideoStreamInputStateProvider input_state_provider_; rtc::scoped_refptr<FakeResource> resource_; rtc::scoped_refptr<FakeResource> other_resource_; + FakeAdaptationConstraint adaptation_constraint_; + FakeAdaptationListener adaptation_listener_; std::unique_ptr<ResourceAdaptationProcessor> processor_; - ResourceAdaptationProcessorListenerForTesting processor_listener_; + VideoSourceRestrictionsListenerForTesting restrictions_listener_; }; } // namespace TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) { - resource_adaptation_queue_.SendTask( - [this] { - EXPECT_EQ(DegradationPreference::DISABLED, - processor_->degradation_preference()); - EXPECT_EQ(DegradationPreference::DISABLED, - processor_->effective_degradation_preference()); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - processor_->StartResourceAdaptation(); - // Adaptation does not happen when disabled. - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - }, - RTC_FROM_HERE); + EXPECT_EQ(DegradationPreference::DISABLED, + processor_->degradation_preference()); + EXPECT_EQ(DegradationPreference::DISABLED, + processor_->effective_degradation_preference()); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adaptation does not happen when disabled. + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); } TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - // Adaptation does not happen if input is insufficient. - // When frame size is missing (OnFrameSizeObserved not called yet). - input_state_provider_.OnHasInputChanged(true); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - // When "has input" is missing. - SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - // Note: frame rate cannot be missing, if unset it is 0. - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + // Adaptation does not happen if input is insufficient. + // When frame size is missing (OnFrameSizeObserved not called yet). + input_state_provider_.OnHasInputChanged(true); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + // When "has input" is missing. + SetInputStates(false, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + // Note: frame rate cannot be missing, if unset it is 0. } // These tests verify that restrictions are applied, but not exactly how much @@ -172,273 +183,558 @@ TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) { // restrictions. For that, see video_stream_adapter_unittest.cc. TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingResolutionInMaintainFrameRate) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_TRUE(processor_listener_.restrictions() - .max_pixels_per_frame() - .has_value()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + EXPECT_TRUE( + restrictions_listener_.restrictions().max_pixels_per_frame().has_value()); } TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingFrameRateInMaintainResolution) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_RESOLUTION); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_TRUE( - processor_listener_.restrictions().max_frame_rate().has_value()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + EXPECT_TRUE( + restrictions_listener_.restrictions().max_frame_rate().has_value()); } TEST_F(ResourceAdaptationProcessorTest, OveruseTriggersRestrictingFrameRateAndResolutionInBalanced) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference(DegradationPreference::BALANCED); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - // Adapting multiple times eventually resticts both frame rate and - // resolution. Exactly many times we need to adapt depends on - // BalancedDegradationSettings, VideoStreamAdapter and default input - // states. This test requires it to be achieved within 4 adaptations. - for (size_t i = 0; i < 4; ++i) { - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(i + 1, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - } - EXPECT_TRUE(processor_listener_.restrictions() - .max_pixels_per_frame() - .has_value()); - EXPECT_TRUE( - processor_listener_.restrictions().max_frame_rate().has_value()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference(DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapting multiple times eventually resticts both frame rate and + // resolution. Exactly many times we need to adapt depends on + // BalancedDegradationSettings, VideoStreamAdapter and default input + // states. This test requires it to be achieved within 4 adaptations. + for (size_t i = 0; i < 4; ++i) { + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(i + 1, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + } + EXPECT_TRUE( + restrictions_listener_.restrictions().max_pixels_per_frame().has_value()); + EXPECT_TRUE( + restrictions_listener_.restrictions().max_frame_rate().has_value()); } TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - // If we don't restrict the source then adaptation will not happen again - // due to "awaiting previous adaptation". This prevents "double-adapt". - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + // If we don't restrict the source then adaptation will not happen again + // due to "awaiting previous adaptation". This prevents "double-adapt". + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); } TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); } TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(2u, processor_listener_.restrictions_updated_count()); - EXPECT_EQ(VideoSourceRestrictions(), - processor_listener_.restrictions()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2u, restrictions_listener_.restrictions_updated_count()); + EXPECT_EQ(VideoSourceRestrictions(), restrictions_listener_.restrictions()); } TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - // Adapt down so that we can adapt up. - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - // Adapting up is prevented. - resource_->set_is_adaptation_up_allowed(false); - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapt down so that we can adapt up. + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + // Adapting up is prevented. + adaptation_constraint_.set_is_adaptation_up_allowed(false); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); } TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNeverAdaptedDown) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - RestrictSource(processor_listener_.restrictions()); - - // Other resource signals under-use - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + RestrictSource(restrictions_listener_.restrictions()); + + // Other resource signals under-use + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); } TEST_F(ResourceAdaptationProcessorTest, ResourcesCanNotAdaptUpIfNotAdaptedDownAfterReset) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - - processor_->ResetVideoSourceRestrictions(); - EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - other_resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - - // resource_ did not overuse after we reset the restrictions, so adapt - // up should be disallowed. - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); + + processor_->ResetVideoSourceRestrictions(); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // resource_ did not overuse after we reset the restrictions, so adapt + // up should be disallowed. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); +} + +TEST_F(ResourceAdaptationProcessorTest, OnlyMostLimitedResourceMayAdaptUp) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // |other_resource_| is most limited, resource_ can't adapt up. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // |resource_| and |other_resource_| are now most limited, so both must + // signal underuse to adapt up. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); } TEST_F(ResourceAdaptationProcessorTest, MultipleResourcesCanTriggerMultipleAdaptations) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(3, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - // Does not trigger adaptation since resource has no adaptations left. - resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(2, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // resource_ is not most limited so can't adapt from underuse. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // resource_ is still not most limited so can't adapt from underuse. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // However it will be after overuse + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // Now other_resource_ can't adapt up as it is not most restricted. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(3, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + // resource_ is limited at 3 adaptations and other_resource_ 2. + // With the most limited resource signalling underuse in the following + // order we get back to unrestricted video. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // Both resource_ and other_resource_ are most limited. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // Again both are most limited. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); +} + +TEST_F(ResourceAdaptationProcessorTest, + MostLimitedResourceAdaptationWorksAfterChangingDegradataionPreference) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + // Adapt down until we can't anymore. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + int last_total = restrictions_listener_.adaptation_counters().Total(); + + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + // resource_ can not adapt up since we have never reduced FPS. + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(last_total, restrictions_listener_.adaptation_counters().Total()); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(last_total + 1, + restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + // other_resource_ is most limited so should be able to adapt up. + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(last_total, restrictions_listener_.adaptation_counters().Total()); } TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, resource_->num_adaptations_applied()); - }, - RTC_FROM_HERE); + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1u, adaptation_listener_.num_adaptations_applied()); } -TEST_F(ResourceAdaptationProcessorTest, AdaptingClearsResourceUsageState) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(1u, processor_listener_.restrictions_updated_count()); - EXPECT_FALSE(resource_->usage_state().has_value()); - }, - RTC_FROM_HERE); +TEST_F(ResourceAdaptationProcessorTest, + AdaptsDownWhenOtherResourceIsAlwaysUnderused) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + // Does not trigger adapataion because there's no restriction. + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + // Adapts down even if other resource asked for adapting up. + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + RestrictSource(restrictions_listener_.restrictions()); + other_resource_->SetUsageState(ResourceUsageState::kUnderuse); + // Doesn't adapt up because adaptation is due to another resource. + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); } TEST_F(ResourceAdaptationProcessorTest, - FailingAdaptingAlsoClearsResourceUsageState) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference(DegradationPreference::DISABLED); - processor_->StartResourceAdaptation(); - resource_->set_usage_state(ResourceUsageState::kOveruse); - EXPECT_EQ(0u, processor_listener_.restrictions_updated_count()); - EXPECT_FALSE(resource_->usage_state().has_value()); - }, - RTC_FROM_HERE); + TriggerOveruseNotOnAdaptationTaskQueue) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + resource_task_queue.PostTask(ToQueuedTask( + [&]() { resource_->SetUsageState(ResourceUsageState::kOveruse); })); + + EXPECT_EQ_WAIT(1u, restrictions_listener_.restrictions_updated_count(), + kDefaultTimeoutMs); } TEST_F(ResourceAdaptationProcessorTest, - AdaptsDownWhenOtherResourceIsAlwaysUnderused) { - resource_adaptation_queue_.SendTask( - [this] { - processor_->SetDegradationPreference( - DegradationPreference::MAINTAIN_FRAMERATE); - processor_->StartResourceAdaptation(); - SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - // Does not trigger adapataion because there's no restriction. - EXPECT_EQ(0, processor_listener_.adaptation_counters().Total()); - - RestrictSource(processor_listener_.restrictions()); - resource_->set_usage_state(ResourceUsageState::kOveruse); - // Adapts down even if other resource asked for adapting up. - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - - RestrictSource(processor_listener_.restrictions()); - other_resource_->set_usage_state(ResourceUsageState::kUnderuse); - // Doesn't adapt up because adaptation is due to another resource. - EXPECT_EQ(1, processor_listener_.adaptation_counters().Total()); - RestrictSource(processor_listener_.restrictions()); - }, - RTC_FROM_HERE); + DestroyProcessorWhileResourceListenerDelegateHasTaskInFlight) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + // Wait for |resource_| to signal oversue first so we know that the delegate + // has passed it on to the processor's task queue. + rtc::Event resource_event; + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + resource_task_queue.PostTask(ToQueuedTask([&]() { + resource_->SetUsageState(ResourceUsageState::kOveruse); + resource_event.Set(); + })); + + EXPECT_TRUE(resource_event.Wait(kDefaultTimeoutMs)); + // Now destroy the processor while handling the overuse is in flight. + DestroyProcessor(); + + // Because the processor was destroyed by the time the delegate's task ran, + // the overuse signal must not have been handled. + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); +} + +TEST_F(ResourceAdaptationProcessorTest, + ResourceOveruseIgnoredWhenSignalledDuringRemoval) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + rtc::Event overuse_event; + TaskQueueForTest resource_task_queue("ResourceTaskQueue"); + // Queues task for |resource_| overuse while |processor_| is still listening. + resource_task_queue.PostTask(ToQueuedTask([&]() { + resource_->SetUsageState(ResourceUsageState::kOveruse); + overuse_event.Set(); + })); + EXPECT_TRUE(overuse_event.Wait(kDefaultTimeoutMs)); + // Once we know the overuse task is queued, remove |resource_| so that + // |processor_| is not listening to it. + processor_->RemoveResource(resource_); + + // Runs the queued task so |processor_| gets signalled kOveruse from + // |resource_| even though |processor_| was not listening. + WaitUntilTaskQueueIdle(); + + // No restrictions should change even though |resource_| signaled |kOveruse|. + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingOnlyAdaptedResourceResetsAdaptation) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + RestrictSource(restrictions_listener_.restrictions()); + + processor_->RemoveResource(resource_); + EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsAdaptationToNextLimitedLevel) { + processor_->SetDegradationPreference(DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited |resource_| should revert us back to + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsAdaptationIfInputStateUnchanged) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + // Overuse twice and underuse once. After the underuse we don't restrict the + // source. Normally this would block future underuses. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited |resource_| should revert us back to, even though we + // did not call RestrictSource() after |resource_| was overused. Normally + // adaptation for MAINTAIN_FRAMERATE would be blocked here but for removal we + // allow this anyways. + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingResourceNotMostLimitedHasNoEffectOnLimitations) { + processor_->SetDegradationPreference(DegradationPreference::BALANCED); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + VideoSourceRestrictions current_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters current_counters = + restrictions_listener_.adaptation_counters(); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Removing most limited |resource_| should revert us back to + processor_->RemoveResource(other_resource_); + EXPECT_EQ(current_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(current_counters, restrictions_listener_.adaptation_counters()); + + // Delete |other_resource_| for cleanup. + other_resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceAfterSwitchingDegradationPreferences) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_RESOLUTION); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + // Revert to |other_resource_| when removing |resource_| even though the + // degradation preference was different when it was overused. + processor_->RemoveResource(resource_); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // After switching back to MAINTAIN_FRAMERATE, the next most limited settings + // are restored. + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingMostLimitedResourceSetsNextLimitationsInDisabled) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + VideoSourceRestrictions next_limited_restrictions = + restrictions_listener_.restrictions(); + VideoAdaptationCounters next_limited_counters = + restrictions_listener_.adaptation_counters(); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total()); + + processor_->SetDegradationPreference(DegradationPreference::DISABLED); + + // Revert to |other_resource_| when removing |resource_| even though the + // current degradataion preference is disabled. + processor_->RemoveResource(resource_); + + // After switching back to MAINTAIN_FRAMERATE, the next most limited settings + // are restored. + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions()); + EXPECT_EQ(next_limited_counters, + restrictions_listener_.adaptation_counters()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovedResourceSignalsIgnoredByProcessor) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + processor_->RemoveResource(resource_); + resource_->SetUsageState(ResourceUsageState::kOveruse); + EXPECT_EQ(0u, restrictions_listener_.restrictions_updated_count()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; +} + +TEST_F(ResourceAdaptationProcessorTest, + RemovingResourceWhenMultipleMostLimtedHasNoEffect) { + processor_->SetDegradationPreference( + DegradationPreference::MAINTAIN_FRAMERATE); + SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize); + + other_resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + // Adapt |resource_| up and then down so that both resource's are most + // limited at 1 adaptation. + resource_->SetUsageState(ResourceUsageState::kOveruse); + RestrictSource(restrictions_listener_.restrictions()); + resource_->SetUsageState(ResourceUsageState::kUnderuse); + RestrictSource(restrictions_listener_.restrictions()); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + // Removing |resource_| has no effect since both |resource_| and + // |other_resource_| are most limited. + processor_->RemoveResource(resource_); + EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); + + // Delete |resource_| for cleanup. + resource_ = nullptr; } } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/resource_unittest.cc b/chromium/third_party/webrtc/call/adaptation/resource_unittest.cc index 8f3ae32dca6..a2291dfdce8 100644 --- a/chromium/third_party/webrtc/call/adaptation/resource_unittest.cc +++ b/chromium/third_party/webrtc/call/adaptation/resource_unittest.cc @@ -8,14 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "call/adaptation/resource.h" +#include "api/adaptation/resource.h" #include <memory> #include "api/scoped_refptr.h" #include "call/adaptation/test/fake_resource.h" -#include "rtc_base/event.h" -#include "rtc_base/task_queue_for_test.h" +#include "call/adaptation/test/mock_resource_listener.h" #include "test/gmock.h" #include "test/gtest.h" @@ -24,56 +23,33 @@ namespace webrtc { using ::testing::_; using ::testing::StrictMock; -class MockResourceListener : public ResourceListener { - public: - MOCK_METHOD(void, - OnResourceUsageStateMeasured, - (rtc::scoped_refptr<Resource> resource)); -}; - class ResourceTest : public ::testing::Test { public: - ResourceTest() - : resource_adaptation_queue_("ResourceAdaptationQueue"), - encoder_queue_("EncoderQueue"), - fake_resource_(new FakeResource("FakeResource")) { - fake_resource_->Initialize(&encoder_queue_, &resource_adaptation_queue_); - } + ResourceTest() : fake_resource_(FakeResource::Create("FakeResource")) {} protected: - const std::unique_ptr<TaskQueueFactory> task_queue_factory_; - TaskQueueForTest resource_adaptation_queue_; - TaskQueueForTest encoder_queue_; rtc::scoped_refptr<FakeResource> fake_resource_; }; TEST_F(ResourceTest, RegisteringListenerReceivesCallbacks) { - resource_adaptation_queue_.SendTask( - [this] { - StrictMock<MockResourceListener> resource_listener; - fake_resource_->SetResourceListener(&resource_listener); - EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)) - .Times(1) - .WillOnce([](rtc::scoped_refptr<Resource> resource) { - EXPECT_EQ(ResourceUsageState::kOveruse, resource->usage_state()); - }); - fake_resource_->set_usage_state(ResourceUsageState::kOveruse); - fake_resource_->SetResourceListener(nullptr); - }, - RTC_FROM_HERE); + StrictMock<MockResourceListener> resource_listener; + fake_resource_->SetResourceListener(&resource_listener); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + fake_resource_->SetUsageState(ResourceUsageState::kOveruse); + fake_resource_->SetResourceListener(nullptr); } TEST_F(ResourceTest, UnregisteringListenerStopsCallbacks) { - resource_adaptation_queue_.SendTask( - [this] { - StrictMock<MockResourceListener> resource_listener; - fake_resource_->SetResourceListener(&resource_listener); - fake_resource_->SetResourceListener(nullptr); - EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_)) - .Times(0); - fake_resource_->set_usage_state(ResourceUsageState::kOveruse); - }, - RTC_FROM_HERE); + StrictMock<MockResourceListener> resource_listener; + fake_resource_->SetResourceListener(&resource_listener); + fake_resource_->SetResourceListener(nullptr); + EXPECT_CALL(resource_listener, OnResourceUsageStateMeasured(_, _)).Times(0); + fake_resource_->SetUsageState(ResourceUsageState::kOveruse); } } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_constraint.cc b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_constraint.cc new file mode 100644 index 00000000000..983885e58ac --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_constraint.cc @@ -0,0 +1,39 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/test/fake_adaptation_constraint.h" + +#include <utility> + +namespace webrtc { + +FakeAdaptationConstraint::FakeAdaptationConstraint(std::string name) + : name_(std::move(name)), is_adaptation_up_allowed_(true) {} + +FakeAdaptationConstraint::~FakeAdaptationConstraint() {} + +void FakeAdaptationConstraint::set_is_adaptation_up_allowed( + bool is_adaptation_up_allowed) { + is_adaptation_up_allowed_ = is_adaptation_up_allowed; +} + +std::string FakeAdaptationConstraint::Name() const { + return name_; +} + +bool FakeAdaptationConstraint::IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr<Resource> reason_resource) const { + return is_adaptation_up_allowed_; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_constraint.h b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_constraint.h new file mode 100644 index 00000000000..74637f48fd0 --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_constraint.h @@ -0,0 +1,42 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_FAKE_ADAPTATION_CONSTRAINT_H_ +#define CALL_ADAPTATION_TEST_FAKE_ADAPTATION_CONSTRAINT_H_ + +#include <string> + +#include "call/adaptation/adaptation_constraint.h" + +namespace webrtc { + +class FakeAdaptationConstraint : public AdaptationConstraint { + public: + explicit FakeAdaptationConstraint(std::string name); + ~FakeAdaptationConstraint() override; + + void set_is_adaptation_up_allowed(bool is_adaptation_up_allowed); + + // AdaptationConstraint implementation. + std::string Name() const override; + bool IsAdaptationUpAllowed( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr<Resource> reason_resource) const override; + + private: + const std::string name_; + bool is_adaptation_up_allowed_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_FAKE_ADAPTATION_CONSTRAINT_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_listener.cc b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_listener.cc new file mode 100644 index 00000000000..7feecd63676 --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_listener.cc @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "call/adaptation/test/fake_adaptation_listener.h" + +namespace webrtc { + +FakeAdaptationListener::FakeAdaptationListener() + : num_adaptations_applied_(0) {} + +FakeAdaptationListener::~FakeAdaptationListener() {} + +size_t FakeAdaptationListener::num_adaptations_applied() const { + return num_adaptations_applied_; +} + +void FakeAdaptationListener::OnAdaptationApplied( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr<Resource> reason_resource) { + ++num_adaptations_applied_; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_listener.h b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_listener.h new file mode 100644 index 00000000000..c60ba3089bc --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_adaptation_listener.h @@ -0,0 +1,38 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_FAKE_ADAPTATION_LISTENER_H_ +#define CALL_ADAPTATION_TEST_FAKE_ADAPTATION_LISTENER_H_ + +#include "call/adaptation/adaptation_listener.h" + +namespace webrtc { + +class FakeAdaptationListener : public AdaptationListener { + public: + FakeAdaptationListener(); + ~FakeAdaptationListener() override; + + size_t num_adaptations_applied() const; + + // AdaptationListener implementation. + void OnAdaptationApplied( + const VideoStreamInputState& input_state, + const VideoSourceRestrictions& restrictions_before, + const VideoSourceRestrictions& restrictions_after, + rtc::scoped_refptr<Resource> reason_resource) override; + + private: + size_t num_adaptations_applied_; +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_FAKE_ADAPTATION_LISTENER_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_frame_rate_provider.h b/chromium/third_party/webrtc/call/adaptation/test/fake_frame_rate_provider.h index a08e1623217..3638f478f37 100644 --- a/chromium/third_party/webrtc/call/adaptation/test/fake_frame_rate_provider.h +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_frame_rate_provider.h @@ -21,29 +21,41 @@ namespace webrtc { class MockVideoStreamEncoderObserver : public VideoStreamEncoderObserver { public: - MOCK_METHOD2(OnEncodedFrameTimeMeasured, void(int, int)); - MOCK_METHOD2(OnIncomingFrame, void(int, int)); - MOCK_METHOD2(OnSendEncodedImage, - void(const EncodedImage&, const CodecSpecificInfo*)); - MOCK_METHOD1(OnEncoderImplementationChanged, void(const std::string&)); - MOCK_METHOD1(OnFrameDropped, void(DropReason)); - MOCK_METHOD2(OnEncoderReconfigured, - void(const VideoEncoderConfig&, - const std::vector<VideoStream>&)); - MOCK_METHOD3(OnAdaptationChanged, - void(VideoAdaptationReason, - const VideoAdaptationCounters&, - const VideoAdaptationCounters&)); - MOCK_METHOD0(ClearAdaptationStats, void()); - MOCK_METHOD2(UpdateAdaptationSettings, - void(AdaptationSettings, AdaptationSettings)); - MOCK_METHOD0(OnMinPixelLimitReached, void()); - MOCK_METHOD0(OnInitialQualityResolutionAdaptDown, void()); - MOCK_METHOD1(OnSuspendChange, void(bool)); - MOCK_METHOD2(OnBitrateAllocationUpdated, - void(const VideoCodec&, const VideoBitrateAllocation&)); - MOCK_METHOD1(OnEncoderInternalScalerUpdate, void(bool)); - MOCK_CONST_METHOD0(GetInputFrameRate, int()); + MOCK_METHOD(void, OnEncodedFrameTimeMeasured, (int, int), (override)); + MOCK_METHOD(void, OnIncomingFrame, (int, int), (override)); + MOCK_METHOD(void, + OnSendEncodedImage, + (const EncodedImage&, const CodecSpecificInfo*), + (override)); + MOCK_METHOD(void, + OnEncoderImplementationChanged, + (const std::string&), + (override)); + MOCK_METHOD(void, OnFrameDropped, (DropReason), (override)); + MOCK_METHOD(void, + OnEncoderReconfigured, + (const VideoEncoderConfig&, const std::vector<VideoStream>&), + (override)); + MOCK_METHOD(void, + OnAdaptationChanged, + (VideoAdaptationReason, + const VideoAdaptationCounters&, + const VideoAdaptationCounters&), + (override)); + MOCK_METHOD(void, ClearAdaptationStats, (), (override)); + MOCK_METHOD(void, + UpdateAdaptationSettings, + (AdaptationSettings, AdaptationSettings), + (override)); + MOCK_METHOD(void, OnMinPixelLimitReached, (), (override)); + MOCK_METHOD(void, OnInitialQualityResolutionAdaptDown, (), (override)); + MOCK_METHOD(void, OnSuspendChange, (bool), (override)); + MOCK_METHOD(void, + OnBitrateAllocationUpdated, + (const VideoCodec&, const VideoBitrateAllocation&), + (override)); + MOCK_METHOD(void, OnEncoderInternalScalerUpdate, (bool), (override)); + MOCK_METHOD(int, GetInputFrameRate, (), (const, override)); }; class FakeFrameRateProvider : public MockVideoStreamEncoderObserver { diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_resource.cc b/chromium/third_party/webrtc/call/adaptation/test/fake_resource.cc index 4c0a129d04e..fa69e886bfc 100644 --- a/chromium/third_party/webrtc/call/adaptation/test/fake_resource.cc +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_resource.cc @@ -10,44 +10,35 @@ #include "call/adaptation/test/fake_resource.h" +#include <algorithm> #include <utility> +#include "rtc_base/ref_counted_object.h" + namespace webrtc { +// static +rtc::scoped_refptr<FakeResource> FakeResource::Create(std::string name) { + return new rtc::RefCountedObject<FakeResource>(name); +} + FakeResource::FakeResource(std::string name) - : rtc::RefCountedObject<Resource>(), - name_(std::move(name)), - is_adaptation_up_allowed_(true), - num_adaptations_applied_(0) {} + : Resource(), name_(std::move(name)), listener_(nullptr) {} FakeResource::~FakeResource() {} -void FakeResource::set_usage_state(ResourceUsageState usage_state) { - OnResourceUsageStateMeasured(usage_state); -} - -void FakeResource::set_is_adaptation_up_allowed(bool is_adaptation_up_allowed) { - is_adaptation_up_allowed_ = is_adaptation_up_allowed; -} - -size_t FakeResource::num_adaptations_applied() const { - return num_adaptations_applied_; +void FakeResource::SetUsageState(ResourceUsageState usage_state) { + if (listener_) { + listener_->OnResourceUsageStateMeasured(this, usage_state); + } } -bool FakeResource::IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) const { - return is_adaptation_up_allowed_; +std::string FakeResource::Name() const { + return name_; } -void FakeResource::OnAdaptationApplied( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) { - ++num_adaptations_applied_; +void FakeResource::SetResourceListener(ResourceListener* listener) { + listener_ = listener; } } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/test/fake_resource.h b/chromium/third_party/webrtc/call/adaptation/test/fake_resource.h index beaca546144..e88d97db7a0 100644 --- a/chromium/third_party/webrtc/call/adaptation/test/fake_resource.h +++ b/chromium/third_party/webrtc/call/adaptation/test/fake_resource.h @@ -12,39 +12,31 @@ #define CALL_ADAPTATION_TEST_FAKE_RESOURCE_H_ #include <string> +#include <vector> -#include "call/adaptation/resource.h" -#include "rtc_base/ref_counted_object.h" +#include "absl/types/optional.h" +#include "api/adaptation/resource.h" +#include "api/scoped_refptr.h" namespace webrtc { // Fake resource used for testing. -class FakeResource : public rtc::RefCountedObject<Resource> { +class FakeResource : public Resource { public: + static rtc::scoped_refptr<FakeResource> Create(std::string name); + explicit FakeResource(std::string name); ~FakeResource() override; - void set_usage_state(ResourceUsageState usage_state); - void set_is_adaptation_up_allowed(bool is_adaptation_up_allowed); - size_t num_adaptations_applied() const; + void SetUsageState(ResourceUsageState usage_state); // Resource implementation. - std::string name() const override { return name_; } - bool IsAdaptationUpAllowed( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) const override; - void OnAdaptationApplied( - const VideoStreamInputState& input_state, - const VideoSourceRestrictions& restrictions_before, - const VideoSourceRestrictions& restrictions_after, - rtc::scoped_refptr<Resource> reason_resource) override; + std::string Name() const override; + void SetResourceListener(ResourceListener* listener) override; private: const std::string name_; - bool is_adaptation_up_allowed_; - size_t num_adaptations_applied_; + ResourceListener* listener_; }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/test/mock_resource_listener.h b/chromium/third_party/webrtc/call/adaptation/test/mock_resource_listener.h new file mode 100644 index 00000000000..f0f998f2e33 --- /dev/null +++ b/chromium/third_party/webrtc/call/adaptation/test/mock_resource_listener.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef CALL_ADAPTATION_TEST_MOCK_RESOURCE_LISTENER_H_ +#define CALL_ADAPTATION_TEST_MOCK_RESOURCE_LISTENER_H_ + +#include "api/adaptation/resource.h" + +#include "test/gmock.h" + +namespace webrtc { + +class MockResourceListener : public ResourceListener { + public: + MOCK_METHOD(void, + OnResourceUsageStateMeasured, + (rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state), + (override)); +}; + +} // namespace webrtc + +#endif // CALL_ADAPTATION_TEST_MOCK_RESOURCE_LISTENER_H_ diff --git a/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.cc b/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.cc index 6fbdcb42a66..e9d6c26137b 100644 --- a/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.cc +++ b/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.cc @@ -13,6 +13,7 @@ #include <limits> #include "rtc_base/checks.h" +#include "rtc_base/strings/string_builder.h" namespace webrtc { @@ -36,6 +37,19 @@ VideoSourceRestrictions::VideoSourceRestrictions( RTC_DCHECK(!max_frame_rate_.has_value() || max_frame_rate_.value() > 0.0); } +std::string VideoSourceRestrictions::ToString() const { + rtc::StringBuilder ss; + ss << "{"; + if (max_frame_rate_) + ss << " max_fps=" << max_frame_rate_.value(); + if (max_pixels_per_frame_) + ss << " max_pixels_per_frame=" << max_pixels_per_frame_.value(); + if (target_pixels_per_frame_) + ss << " target_pixels_per_frame=" << target_pixels_per_frame_.value(); + ss << " }"; + return ss.Release(); +} + const absl::optional<size_t>& VideoSourceRestrictions::max_pixels_per_frame() const { return max_pixels_per_frame_; diff --git a/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.h b/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.h index 506bae61334..7f79a48e5d4 100644 --- a/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.h +++ b/chromium/third_party/webrtc/call/adaptation/video_source_restrictions.h @@ -11,6 +11,7 @@ #ifndef CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ #define CALL_ADAPTATION_VIDEO_SOURCE_RESTRICTIONS_H_ +#include <string> #include <utility> #include "absl/types/optional.h" @@ -38,6 +39,8 @@ class VideoSourceRestrictions { return !(*this == rhs); } + std::string ToString() const; + // The source must produce a resolution less than or equal to // max_pixels_per_frame(). const absl::optional<size_t>& max_pixels_per_frame() const; diff --git a/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.cc b/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.cc index 4ebe00fb0c3..4bf236fe714 100644 --- a/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.cc +++ b/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.cc @@ -111,8 +111,27 @@ int GetHigherResolutionThan(int pixel_count) { : std::numeric_limits<int>::max(); } -Adaptation::Step::Step(StepType type, int target) - : type(type), target(target) {} +// static +const char* Adaptation::StatusToString(Adaptation::Status status) { + switch (status) { + case Adaptation::Status::kValid: + return "kValid"; + case Adaptation::Status::kLimitReached: + return "kLimitReached"; + case Adaptation::Status::kAwaitingPreviousAdaptation: + return "kAwaitingPreviousAdaptation"; + } +} + +Adaptation::Step::Step(StepType type, int target) : type(type), target(target) { + RTC_DCHECK_NE(type, Adaptation::StepType::kForce); +} + +Adaptation::Step::Step(VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters) + : type(Adaptation::StepType::kForce), + restrictions(restrictions), + counters(counters) {} Adaptation::Adaptation(int validation_id, Step step) : validation_id_(validation_id), @@ -176,13 +195,19 @@ class VideoStreamAdapter::VideoSourceRestrictor { adaptations_ = VideoAdaptationCounters(); } + void ForceRestrictions(const VideoSourceRestrictions& restrictions, + const VideoAdaptationCounters& counters) { + source_restrictions_ = restrictions; + adaptations_ = counters; + } + void set_min_pixels_per_frame(int min_pixels_per_frame) { min_pixels_per_frame_ = min_pixels_per_frame; } int min_pixels_per_frame() const { return min_pixels_per_frame_; } - bool CanDecreaseResolutionTo(int target_pixels) { + bool CanDecreaseResolutionTo(int target_pixels) const { int max_pixels_per_frame = rtc::dchecked_cast<int>( source_restrictions_.max_pixels_per_frame().value_or( std::numeric_limits<int>::max())); @@ -190,7 +215,7 @@ class VideoStreamAdapter::VideoSourceRestrictor { target_pixels >= min_pixels_per_frame_; } - bool CanIncreaseResolutionTo(int target_pixels) { + bool CanIncreaseResolutionTo(int target_pixels) const { int max_pixels_wanted = GetIncreasedMaxPixelsWanted(target_pixels); int max_pixels_per_frame = rtc::dchecked_cast<int>( source_restrictions_.max_pixels_per_frame().value_or( @@ -198,14 +223,14 @@ class VideoStreamAdapter::VideoSourceRestrictor { return max_pixels_wanted > max_pixels_per_frame; } - bool CanDecreaseFrameRateTo(int max_frame_rate) { + bool CanDecreaseFrameRateTo(int max_frame_rate) const { const int fps_wanted = std::max(kMinFrameRateFps, max_frame_rate); return fps_wanted < rtc::dchecked_cast<int>( source_restrictions_.max_frame_rate().value_or( std::numeric_limits<int>::max())); } - bool CanIncreaseFrameRateTo(int max_frame_rate) { + bool CanIncreaseFrameRateTo(int max_frame_rate) const { return max_frame_rate > rtc::dchecked_cast<int>( source_restrictions_.max_frame_rate().value_or( std::numeric_limits<int>::max())); @@ -215,13 +240,16 @@ class VideoStreamAdapter::VideoSourceRestrictor { DegradationPreference degradation_preference) { switch (step.type) { case Adaptation::StepType::kIncreaseResolution: - IncreaseResolutionTo(step.target); + RTC_DCHECK(step.target); + IncreaseResolutionTo(step.target.value()); break; case Adaptation::StepType::kDecreaseResolution: - DecreaseResolutionTo(step.target); + RTC_DCHECK(step.target); + DecreaseResolutionTo(step.target.value()); break; case Adaptation::StepType::kIncreaseFrameRate: - IncreaseFrameRateTo(step.target); + RTC_DCHECK(step.target); + IncreaseFrameRateTo(step.target.value()); // TODO(https://crbug.com/webrtc/11222): Don't adapt in two steps. // GetAdaptationUp() should tell us the correct value, but BALANCED // logic in DecrementFramerate() makes it hard to predict whether this @@ -235,7 +263,13 @@ class VideoStreamAdapter::VideoSourceRestrictor { } break; case Adaptation::StepType::kDecreaseFrameRate: - DecreaseFrameRateTo(step.target); + RTC_DCHECK(step.target); + DecreaseFrameRateTo(step.target.value()); + break; + case Adaptation::StepType::kForce: + RTC_DCHECK(step.restrictions); + RTC_DCHECK(step.counters); + ForceRestrictions(step.restrictions.value(), step.counters.value()); break; } } @@ -501,19 +535,23 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const { } } -VideoSourceRestrictions VideoStreamAdapter::PeekNextRestrictions( - const Adaptation& adaptation) const { +VideoStreamAdapter::RestrictionsWithCounters +VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const { RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); + RTC_LOG(LS_INFO) << "PeekNextRestrictions called"; if (adaptation.status() != Adaptation::Status::kValid) - return source_restrictor_->source_restrictions(); + return {source_restrictor_->source_restrictions(), + source_restrictor_->adaptation_counters()}; VideoSourceRestrictor restrictor_copy = *source_restrictor_; restrictor_copy.ApplyAdaptationStep(adaptation.step(), degradation_preference_); - return restrictor_copy.source_restrictions(); + return {restrictor_copy.source_restrictions(), + restrictor_copy.adaptation_counters()}; } void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) { RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); + RTC_LOG(LS_INFO) << "ApplyAdaptation called"; if (adaptation.status() != Adaptation::Status::kValid) return; // Remember the input pixels and fps of this adaptation. Used to avoid @@ -526,4 +564,12 @@ void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) { degradation_preference_); } +Adaptation VideoStreamAdapter::GetAdaptationTo( + const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions) const { + // Adapts up/down from the current levels so counters are equal. + return Adaptation(adaptation_validation_id_, + Adaptation::Step(restrictions, counters)); +} + } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.h b/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.h index f313e6bed6c..3a56f4f7c52 100644 --- a/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.h +++ b/chromium/third_party/webrtc/call/adaptation/video_stream_adapter.h @@ -14,9 +14,9 @@ #include <memory> #include "absl/types/optional.h" +#include "api/adaptation/resource.h" #include "api/rtp_parameters.h" #include "api/video/video_adaptation_counters.h" -#include "call/adaptation/resource.h" #include "call/adaptation/video_source_restrictions.h" #include "call/adaptation/video_stream_input_state.h" #include "modules/video_coding/utility/quality_scaler.h" @@ -56,6 +56,8 @@ class Adaptation final { kAwaitingPreviousAdaptation, }; + static const char* StatusToString(Status status); + // The status of this Adaptation. To find out how this Adaptation affects // VideoSourceRestrictions, see VideoStreamAdapter::PeekNextRestrictions(). Status status() const; @@ -73,12 +75,22 @@ class Adaptation final { kDecreaseResolution, kIncreaseFrameRate, kDecreaseFrameRate, + kForce }; struct Step { Step(StepType type, int target); + // StepType is kForce + Step(VideoSourceRestrictions restrictions, + VideoAdaptationCounters counters); const StepType type; - const int target; // Pixel or frame rate depending on |type|. + // Pixel or frame rate depending on |type|. + // Only set when |type| is not kForce. + const absl::optional<int> target; + // Only set when |type| is kForce. + const absl::optional<VideoSourceRestrictions> restrictions; + // Only set when |type| is kForce. + const absl::optional<VideoAdaptationCounters> counters; }; // Constructs with a valid adaptation Step. Status is kValid. @@ -127,10 +139,18 @@ class VideoStreamAdapter { // status code indicating the reason why we cannot adapt. Adaptation GetAdaptationUp() const; Adaptation GetAdaptationDown() const; + Adaptation GetAdaptationTo(const VideoAdaptationCounters& counters, + const VideoSourceRestrictions& restrictions) const; + + struct RestrictionsWithCounters { + VideoSourceRestrictions restrictions; + VideoAdaptationCounters adaptation_counters; + }; + // Returns the restrictions that result from applying the adaptation, without // actually applying it. If the adaptation is not valid, current restrictions // are returned. - VideoSourceRestrictions PeekNextRestrictions( + RestrictionsWithCounters PeekNextRestrictions( const Adaptation& adaptation) const; // Updates source_restrictions() based according to the Adaptation. void ApplyAdaptation(const Adaptation& adaptation); diff --git a/chromium/third_party/webrtc/call/adaptation/video_stream_adapter_unittest.cc b/chromium/third_party/webrtc/call/adaptation/video_stream_adapter_unittest.cc index 79247a7837d..49b291c3569 100644 --- a/chromium/third_party/webrtc/call/adaptation/video_stream_adapter_unittest.cc +++ b/chromium/third_party/webrtc/call/adaptation/video_stream_adapter_unittest.cc @@ -686,26 +686,35 @@ TEST(VideoStreamAdapterTest, PeekNextRestrictions) { { Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kLimitReached, adaptation.status()); - EXPECT_EQ(adapter.PeekNextRestrictions(adaptation), + VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters = + adapter.PeekNextRestrictions(adaptation); + EXPECT_EQ(restrictions_with_counters.restrictions, adapter.source_restrictions()); + EXPECT_EQ(0, restrictions_with_counters.adaptation_counters.Total()); } // When we adapt down. { Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - VideoSourceRestrictions next_restrictions = + VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters = adapter.PeekNextRestrictions(adaptation); fake_stream.ApplyAdaptation(adaptation); - EXPECT_EQ(next_restrictions, adapter.source_restrictions()); + EXPECT_EQ(restrictions_with_counters.restrictions, + adapter.source_restrictions()); + EXPECT_EQ(restrictions_with_counters.adaptation_counters, + adapter.adaptation_counters()); } // When we adapt up. { Adaptation adaptation = adapter.GetAdaptationUp(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - VideoSourceRestrictions next_restrictions = + VideoStreamAdapter::RestrictionsWithCounters restrictions_with_counters = adapter.PeekNextRestrictions(adaptation); fake_stream.ApplyAdaptation(adaptation); - EXPECT_EQ(next_restrictions, adapter.source_restrictions()); + EXPECT_EQ(restrictions_with_counters.restrictions, + adapter.source_restrictions()); + EXPECT_EQ(restrictions_with_counters.adaptation_counters, + adapter.adaptation_counters()); } } diff --git a/chromium/third_party/webrtc/call/audio_send_stream.cc b/chromium/third_party/webrtc/call/audio_send_stream.cc index ddcba031a76..765ece7eb9b 100644 --- a/chromium/third_party/webrtc/call/audio_send_stream.cc +++ b/chromium/third_party/webrtc/call/audio_send_stream.cc @@ -75,6 +75,8 @@ std::string AudioSendStream::Config::SendCodecSpec::ToString() const { ss << ", transport_cc_enabled: " << (transport_cc_enabled ? "true" : "false"); ss << ", cng_payload_type: " << (cng_payload_type ? rtc::ToString(*cng_payload_type) : "<unset>"); + ss << ", red_payload_type: " + << (red_payload_type ? rtc::ToString(*red_payload_type) : "<unset>"); ss << ", payload_type: " << payload_type; ss << ", format: " << rtc::ToString(format); ss << '}'; diff --git a/chromium/third_party/webrtc/call/audio_send_stream.h b/chromium/third_party/webrtc/call/audio_send_stream.h index 86cea38938f..d21dff48891 100644 --- a/chromium/third_party/webrtc/call/audio_send_stream.h +++ b/chromium/third_party/webrtc/call/audio_send_stream.h @@ -140,6 +140,7 @@ class AudioSendStream : public AudioSender { bool nack_enabled = false; bool transport_cc_enabled = false; absl::optional<int> cng_payload_type; + absl::optional<int> red_payload_type; // If unset, use the encoder's default target bitrate. absl::optional<int> target_bitrate_bps; }; diff --git a/chromium/third_party/webrtc/call/bitrate_allocator_unittest.cc b/chromium/third_party/webrtc/call/bitrate_allocator_unittest.cc index 1479a4714aa..00fb2369483 100644 --- a/chromium/third_party/webrtc/call/bitrate_allocator_unittest.cc +++ b/chromium/third_party/webrtc/call/bitrate_allocator_unittest.cc @@ -47,7 +47,10 @@ auto AllocationLimitsEq(uint32_t min_allocatable_rate_bps, class MockLimitObserver : public BitrateAllocator::LimitObserver { public: - MOCK_METHOD1(OnAllocationLimitsChanged, void(BitrateAllocationLimits)); + MOCK_METHOD(void, + OnAllocationLimitsChanged, + (BitrateAllocationLimits), + (override)); }; class TestBitrateObserver : public BitrateAllocatorObserver { diff --git a/chromium/third_party/webrtc/call/call.cc b/chromium/third_party/webrtc/call/call.cc index 4068db9f007..0ef2a3a4bc0 100644 --- a/chromium/third_party/webrtc/call/call.cc +++ b/chromium/third_party/webrtc/call/call.cc @@ -25,6 +25,7 @@ #include "audio/audio_receive_stream.h" #include "audio/audio_send_stream.h" #include "audio/audio_state.h" +#include "call/adaptation/broadcast_resource_listener.h" #include "call/bitrate_allocator.h" #include "call/flexfec_receive_stream_impl.h" #include "call/receive_time_calculator.h" @@ -49,8 +50,8 @@ #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/strings/string_builder.h" -#include "rtc_base/synchronization/rw_lock_wrapper.h" #include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/task_utils/pending_task_safety_flag.h" #include "rtc_base/thread_annotations.h" #include "rtc_base/time_utils.h" #include "rtc_base/trace_event.h" @@ -168,6 +169,47 @@ TaskQueueBase* GetCurrentTaskQueueOrThread() { namespace internal { +// Wraps an injected resource in a BroadcastResourceListener and handles adding +// and removing adapter resources to individual VideoSendStreams. +class ResourceVideoSendStreamForwarder { + public: + ResourceVideoSendStreamForwarder( + rtc::scoped_refptr<webrtc::Resource> resource) + : broadcast_resource_listener_(resource) { + broadcast_resource_listener_.StartListening(); + } + ~ResourceVideoSendStreamForwarder() { + RTC_DCHECK(adapter_resources_.empty()); + broadcast_resource_listener_.StopListening(); + } + + rtc::scoped_refptr<webrtc::Resource> Resource() const { + return broadcast_resource_listener_.SourceResource(); + } + + void OnCreateVideoSendStream(VideoSendStream* video_send_stream) { + RTC_DCHECK(adapter_resources_.find(video_send_stream) == + adapter_resources_.end()); + auto adapter_resource = + broadcast_resource_listener_.CreateAdapterResource(); + video_send_stream->AddAdaptationResource(adapter_resource); + adapter_resources_.insert( + std::make_pair(video_send_stream, adapter_resource)); + } + + void OnDestroyVideoSendStream(VideoSendStream* video_send_stream) { + auto it = adapter_resources_.find(video_send_stream); + RTC_DCHECK(it != adapter_resources_.end()); + broadcast_resource_listener_.RemoveAdapterResource(it->second); + adapter_resources_.erase(it); + } + + private: + BroadcastResourceListener broadcast_resource_listener_; + std::map<VideoSendStream*, rtc::scoped_refptr<webrtc::Resource>> + adapter_resources_; +}; + class Call final : public webrtc::Call, public PacketReceiver, public RecoveredPacketReceiver, @@ -177,7 +219,7 @@ class Call final : public webrtc::Call, Call(Clock* clock, const Call::Config& config, std::unique_ptr<RtpTransportControllerSendInterface> transport_send, - std::unique_ptr<ProcessThread> module_process_thread, + rtc::scoped_refptr<SharedModuleThread> module_process_thread, TaskQueueFactory* task_queue_factory); ~Call() override; @@ -212,6 +254,8 @@ class Call final : public webrtc::Call, void DestroyFlexfecReceiveStream( FlexfecReceiveStream* receive_stream) override; + void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override; + RtpTransportControllerSendInterface* GetTransportControllerSend() override; Stats GetStats() const override; @@ -243,54 +287,54 @@ class Call final : public webrtc::Call, private: DeliveryStatus DeliverRtcp(MediaType media_type, const uint8_t* packet, - size_t length); + size_t length) + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); DeliveryStatus DeliverRtp(MediaType media_type, rtc::CopyOnWriteBuffer packet, - int64_t packet_time_us); + int64_t packet_time_us) + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); void ConfigureSync(const std::string& sync_group) - RTC_EXCLUSIVE_LOCKS_REQUIRED(receive_crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); void NotifyBweOfReceivedPacket(const RtpPacketReceived& packet, MediaType media_type) - RTC_SHARED_LOCKS_REQUIRED(receive_crit_); + RTC_SHARED_LOCKS_REQUIRED(worker_thread_); void UpdateSendHistograms(Timestamp first_sent_packet) - RTC_EXCLUSIVE_LOCKS_REQUIRED(&bitrate_crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); void UpdateReceiveHistograms(); void UpdateHistograms(); void UpdateAggregateNetworkState(); void RegisterRateObserver(); - rtc::TaskQueue* network_queue() const { + rtc::TaskQueue* send_transport_queue() const { return transport_send_ptr_->GetWorkerQueue(); } Clock* const clock_; TaskQueueFactory* const task_queue_factory_; + TaskQueueBase* const worker_thread_; const int num_cpu_cores_; - const std::unique_ptr<ProcessThread> module_process_thread_; + const rtc::scoped_refptr<SharedModuleThread> module_process_thread_; const std::unique_ptr<CallStats> call_stats_; const std::unique_ptr<BitrateAllocator> bitrate_allocator_; Call::Config config_; - SequenceChecker configuration_sequence_checker_; - SequenceChecker worker_sequence_checker_; NetworkState audio_network_state_; NetworkState video_network_state_; - bool aggregate_network_up_ RTC_GUARDED_BY(configuration_sequence_checker_); + bool aggregate_network_up_ RTC_GUARDED_BY(worker_thread_); - std::unique_ptr<RWLockWrapper> receive_crit_; // Audio, Video, and FlexFEC receive streams are owned by the client that // creates them. std::set<AudioReceiveStream*> audio_receive_streams_ - RTC_GUARDED_BY(receive_crit_); + RTC_GUARDED_BY(worker_thread_); std::set<VideoReceiveStream2*> video_receive_streams_ - RTC_GUARDED_BY(receive_crit_); + RTC_GUARDED_BY(worker_thread_); std::map<std::string, AudioReceiveStream*> sync_stream_mapping_ - RTC_GUARDED_BY(receive_crit_); + RTC_GUARDED_BY(worker_thread_); // TODO(nisse): Should eventually be injected at creation, // with a single object in the bundled case. @@ -324,25 +368,26 @@ class Call final : public webrtc::Call, const bool use_send_side_bwe; }; std::map<uint32_t, ReceiveRtpConfig> receive_rtp_config_ - RTC_GUARDED_BY(receive_crit_); + RTC_GUARDED_BY(worker_thread_); - std::unique_ptr<RWLockWrapper> send_crit_; // Audio and Video send streams are owned by the client that creates them. std::map<uint32_t, AudioSendStream*> audio_send_ssrcs_ - RTC_GUARDED_BY(send_crit_); + RTC_GUARDED_BY(worker_thread_); std::map<uint32_t, VideoSendStream*> video_send_ssrcs_ - RTC_GUARDED_BY(send_crit_); - std::set<VideoSendStream*> video_send_streams_ RTC_GUARDED_BY(send_crit_); + RTC_GUARDED_BY(worker_thread_); + std::set<VideoSendStream*> video_send_streams_ RTC_GUARDED_BY(worker_thread_); + + // Each forwarder wraps an adaptation resource that was added to the call. + std::vector<std::unique_ptr<ResourceVideoSendStreamForwarder>> + adaptation_resource_forwarders_ RTC_GUARDED_BY(worker_thread_); using RtpStateMap = std::map<uint32_t, RtpState>; - RtpStateMap suspended_audio_send_ssrcs_ - RTC_GUARDED_BY(configuration_sequence_checker_); - RtpStateMap suspended_video_send_ssrcs_ - RTC_GUARDED_BY(configuration_sequence_checker_); + RtpStateMap suspended_audio_send_ssrcs_ RTC_GUARDED_BY(worker_thread_); + RtpStateMap suspended_video_send_ssrcs_ RTC_GUARDED_BY(worker_thread_); using RtpPayloadStateMap = std::map<uint32_t, RtpPayloadState>; RtpPayloadStateMap suspended_video_payload_states_ - RTC_GUARDED_BY(configuration_sequence_checker_); + RTC_GUARDED_BY(worker_thread_); webrtc::RtcEventLog* event_log_; @@ -358,17 +403,14 @@ class Call final : public webrtc::Call, absl::optional<int64_t> first_received_rtp_video_ms_; absl::optional<int64_t> last_received_rtp_video_ms_; - rtc::CriticalSection last_bandwidth_bps_crit_; - uint32_t last_bandwidth_bps_ RTC_GUARDED_BY(&last_bandwidth_bps_crit_); + uint32_t last_bandwidth_bps_ RTC_GUARDED_BY(worker_thread_); // TODO(holmer): Remove this lock once BitrateController no longer calls // OnNetworkChanged from multiple threads. - rtc::CriticalSection bitrate_crit_; - uint32_t min_allocated_send_bitrate_bps_ - RTC_GUARDED_BY(&worker_sequence_checker_); - uint32_t configured_max_padding_bitrate_bps_ RTC_GUARDED_BY(&bitrate_crit_); + uint32_t min_allocated_send_bitrate_bps_ RTC_GUARDED_BY(worker_thread_); + uint32_t configured_max_padding_bitrate_bps_ RTC_GUARDED_BY(worker_thread_); AvgCounter estimated_send_bitrate_kbps_counter_ - RTC_GUARDED_BY(&bitrate_crit_); - AvgCounter pacer_bitrate_kbps_counter_ RTC_GUARDED_BY(&bitrate_crit_); + RTC_GUARDED_BY(worker_thread_); + AvgCounter pacer_bitrate_kbps_counter_ RTC_GUARDED_BY(worker_thread_); ReceiveSideCongestionController receive_side_cc_; @@ -377,6 +419,11 @@ class Call final : public webrtc::Call, const std::unique_ptr<SendDelayStats> video_send_delay_stats_; const int64_t start_ms_; + // Note that |task_safety_| needs to be at a greater scope than the task queue + // owned by |transport_send_| since calls might arrive on the network thread + // while Call is being deleted and the task queue is being torn down. + ScopedTaskSafety task_safety_; + // Caches transport_send_.get(), to avoid racing with destructor. // Note that this is declared before transport_send_ to ensure that it is not // invalidated until no more tasks can be running on the transport_send_ task @@ -386,8 +433,8 @@ class Call final : public webrtc::Call, // last ensures that it is destroyed first and any running tasks are finished. std::unique_ptr<RtpTransportControllerSendInterface> transport_send_; - bool is_target_rate_observer_registered_ - RTC_GUARDED_BY(&configuration_sequence_checker_) = false; + bool is_target_rate_observer_registered_ RTC_GUARDED_BY(worker_thread_) = + false; RTC_DISALLOW_COPY_AND_ASSIGN(Call); }; @@ -407,14 +454,20 @@ std::string Call::Stats::ToString(int64_t time_ms) const { } Call* Call::Create(const Call::Config& config) { - return Create(config, Clock::GetRealTimeClock(), - ProcessThread::Create("ModuleProcessThread"), + rtc::scoped_refptr<SharedModuleThread> call_thread = + SharedModuleThread::Create("ModuleProcessThread", nullptr); + return Create(config, std::move(call_thread)); +} + +Call* Call::Create(const Call::Config& config, + rtc::scoped_refptr<SharedModuleThread> call_thread) { + return Create(config, Clock::GetRealTimeClock(), std::move(call_thread), ProcessThread::Create("PacerThread")); } Call* Call::Create(const Call::Config& config, Clock* clock, - std::unique_ptr<ProcessThread> call_thread, + rtc::scoped_refptr<SharedModuleThread> call_thread, std::unique_ptr<ProcessThread> pacer_thread) { RTC_DCHECK(config.task_queue_factory); return new internal::Call( @@ -426,6 +479,104 @@ Call* Call::Create(const Call::Config& config, std::move(call_thread), config.task_queue_factory); } +class SharedModuleThread::Impl { + public: + Impl(std::unique_ptr<ProcessThread> process_thread, + std::function<void()> on_one_ref_remaining) + : module_thread_(std::move(process_thread)), + on_one_ref_remaining_(std::move(on_one_ref_remaining)) {} + + void EnsureStarted() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + if (started_) + return; + started_ = true; + module_thread_->Start(); + } + + ProcessThread* process_thread() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return module_thread_.get(); + } + + void AddRef() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + ++ref_count_; + } + + rtc::RefCountReleaseStatus Release() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + --ref_count_; + + if (ref_count_ == 0) { + module_thread_->Stop(); + return rtc::RefCountReleaseStatus::kDroppedLastRef; + } + + if (ref_count_ == 1 && on_one_ref_remaining_) { + auto moved_fn = std::move(on_one_ref_remaining_); + // NOTE: after this function returns, chances are that |this| has been + // deleted - do not touch any member variables. + // If the owner of the last reference implements a lambda that releases + // that last reference inside of the callback (which is legal according + // to this implementation), we will recursively enter Release() above, + // call Stop() and release the last reference. + moved_fn(); + } + + return rtc::RefCountReleaseStatus::kOtherRefsRemained; + } + + private: + SequenceChecker sequence_checker_; + mutable int ref_count_ RTC_GUARDED_BY(sequence_checker_) = 0; + std::unique_ptr<ProcessThread> const module_thread_; + std::function<void()> const on_one_ref_remaining_; + bool started_ = false; +}; + +SharedModuleThread::SharedModuleThread( + std::unique_ptr<ProcessThread> process_thread, + std::function<void()> on_one_ref_remaining) + : impl_(std::make_unique<Impl>(std::move(process_thread), + std::move(on_one_ref_remaining))) {} + +SharedModuleThread::~SharedModuleThread() = default; + +// static +rtc::scoped_refptr<SharedModuleThread> SharedModuleThread::Create( + const char* name, + std::function<void()> on_one_ref_remaining) { + return new SharedModuleThread(ProcessThread::Create(name), + std::move(on_one_ref_remaining)); +} + +rtc::scoped_refptr<SharedModuleThread> SharedModuleThread::Create( + std::unique_ptr<ProcessThread> process_thread, + std::function<void()> on_one_ref_remaining) { + return new SharedModuleThread(std::move(process_thread), + std::move(on_one_ref_remaining)); +} + +void SharedModuleThread::EnsureStarted() { + impl_->EnsureStarted(); +} + +ProcessThread* SharedModuleThread::process_thread() { + return impl_->process_thread(); +} + +void SharedModuleThread::AddRef() const { + impl_->AddRef(); +} + +rtc::RefCountReleaseStatus SharedModuleThread::Release() const { + auto ret = impl_->Release(); + if (ret == rtc::RefCountReleaseStatus::kDroppedLastRef) + delete this; + return ret; +} + // This method here to avoid subclasses has to implement this method. // Call perf test will use Internal::Call::CreateVideoSendStream() to inject // FecController. @@ -441,20 +592,19 @@ namespace internal { Call::Call(Clock* clock, const Call::Config& config, std::unique_ptr<RtpTransportControllerSendInterface> transport_send, - std::unique_ptr<ProcessThread> module_process_thread, + rtc::scoped_refptr<SharedModuleThread> module_process_thread, TaskQueueFactory* task_queue_factory) : clock_(clock), task_queue_factory_(task_queue_factory), + worker_thread_(GetCurrentTaskQueueOrThread()), num_cpu_cores_(CpuInfo::DetectNumberOfCores()), module_process_thread_(std::move(module_process_thread)), - call_stats_(new CallStats(clock_, GetCurrentTaskQueueOrThread())), + call_stats_(new CallStats(clock_, worker_thread_)), bitrate_allocator_(new BitrateAllocator(this)), config_(config), audio_network_state_(kNetworkDown), video_network_state_(kNetworkDown), aggregate_network_up_(false), - receive_crit_(RWLockWrapper::CreateRWLock()), - send_crit_(RWLockWrapper::CreateRWLock()), event_log_(config.event_log), received_bytes_per_second_counter_(clock_, nullptr, true), received_audio_bytes_per_second_counter_(clock_, nullptr, true), @@ -473,17 +623,18 @@ Call::Call(Clock* clock, transport_send_(std::move(transport_send)) { RTC_DCHECK(config.event_log != nullptr); RTC_DCHECK(config.trials != nullptr); - worker_sequence_checker_.Detach(); + RTC_DCHECK(worker_thread_->IsCurrent()); call_stats_->RegisterStatsObserver(&receive_side_cc_); - module_process_thread_->RegisterModule( + module_process_thread_->process_thread()->RegisterModule( receive_side_cc_.GetRemoteBitrateEstimator(true), RTC_FROM_HERE); - module_process_thread_->RegisterModule(&receive_side_cc_, RTC_FROM_HERE); + module_process_thread_->process_thread()->RegisterModule(&receive_side_cc_, + RTC_FROM_HERE); } Call::~Call() { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RTC_CHECK(audio_send_ssrcs_.empty()); RTC_CHECK(video_send_ssrcs_.empty()); @@ -491,10 +642,9 @@ Call::~Call() { RTC_CHECK(audio_receive_streams_.empty()); RTC_CHECK(video_receive_streams_.empty()); - module_process_thread_->Stop(); - module_process_thread_->DeRegisterModule( + module_process_thread_->process_thread()->DeRegisterModule( receive_side_cc_.GetRemoteBitrateEstimator(true)); - module_process_thread_->DeRegisterModule(&receive_side_cc_); + module_process_thread_->process_thread()->DeRegisterModule(&receive_side_cc_); call_stats_->DeregisterStatsObserver(&receive_side_cc_); absl::optional<Timestamp> first_sent_packet_ms = @@ -503,7 +653,6 @@ Call::~Call() { // Only update histograms after process threads have been shut down, so that // they won't try to concurrently update stats. if (first_sent_packet_ms) { - rtc::CritScope lock(&bitrate_crit_); UpdateSendHistograms(*first_sent_packet_ms); } @@ -512,7 +661,7 @@ Call::~Call() { } void Call::RegisterRateObserver() { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); if (is_target_rate_observer_registered_) return; @@ -523,11 +672,11 @@ void Call::RegisterRateObserver() { // off being kicked off on request rather than in the ctor. transport_send_ptr_->RegisterTargetTransferRateObserver(this); - module_process_thread_->Start(); + module_process_thread_->EnsureStarted(); } void Call::SetClientBitratePreferences(const BitrateSettings& preferences) { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); GetTransportControllerSend()->SetClientBitratePreferences(preferences); } @@ -609,14 +758,14 @@ void Call::UpdateReceiveHistograms() { } PacketReceiver* Call::Receiver() { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); return this; } webrtc::AudioSendStream* Call::CreateAudioSendStream( const webrtc::AudioSendStream::Config& config) { TRACE_EVENT0("webrtc", "Call::CreateAudioSendStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RegisterRateObserver(); @@ -632,30 +781,26 @@ webrtc::AudioSendStream* Call::CreateAudioSendStream( AudioSendStream* send_stream = new AudioSendStream( clock_, config, config_.audio_state, task_queue_factory_, - module_process_thread_.get(), transport_send_ptr_, + module_process_thread_->process_thread(), transport_send_ptr_, bitrate_allocator_.get(), event_log_, call_stats_->AsRtcpRttStats(), suspended_rtp_state); - { - WriteLockScoped write_lock(*send_crit_); - RTC_DCHECK(audio_send_ssrcs_.find(config.rtp.ssrc) == - audio_send_ssrcs_.end()); - audio_send_ssrcs_[config.rtp.ssrc] = send_stream; - } - { - ReadLockScoped read_lock(*receive_crit_); - for (AudioReceiveStream* stream : audio_receive_streams_) { - if (stream->config().rtp.local_ssrc == config.rtp.ssrc) { - stream->AssociateSendStream(send_stream); - } + RTC_DCHECK(audio_send_ssrcs_.find(config.rtp.ssrc) == + audio_send_ssrcs_.end()); + audio_send_ssrcs_[config.rtp.ssrc] = send_stream; + + for (AudioReceiveStream* stream : audio_receive_streams_) { + if (stream->config().rtp.local_ssrc == config.rtp.ssrc) { + stream->AssociateSendStream(send_stream); } } + UpdateAggregateNetworkState(); return send_stream; } void Call::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) { TRACE_EVENT0("webrtc", "Call::DestroyAudioSendStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RTC_DCHECK(send_stream != nullptr); send_stream->Stop(); @@ -664,19 +809,16 @@ void Call::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) { webrtc::internal::AudioSendStream* audio_send_stream = static_cast<webrtc::internal::AudioSendStream*>(send_stream); suspended_audio_send_ssrcs_[ssrc] = audio_send_stream->GetRtpState(); - { - WriteLockScoped write_lock(*send_crit_); - size_t num_deleted = audio_send_ssrcs_.erase(ssrc); - RTC_DCHECK_EQ(1, num_deleted); - } - { - ReadLockScoped read_lock(*receive_crit_); - for (AudioReceiveStream* stream : audio_receive_streams_) { - if (stream->config().rtp.local_ssrc == ssrc) { - stream->AssociateSendStream(nullptr); - } + + size_t num_deleted = audio_send_ssrcs_.erase(ssrc); + RTC_DCHECK_EQ(1, num_deleted); + + for (AudioReceiveStream* stream : audio_receive_streams_) { + if (stream->config().rtp.local_ssrc == ssrc) { + stream->AssociateSendStream(nullptr); } } + UpdateAggregateNetworkState(); delete send_stream; } @@ -684,29 +826,25 @@ void Call::DestroyAudioSendStream(webrtc::AudioSendStream* send_stream) { webrtc::AudioReceiveStream* Call::CreateAudioReceiveStream( const webrtc::AudioReceiveStream::Config& config) { TRACE_EVENT0("webrtc", "Call::CreateAudioReceiveStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RegisterRateObserver(); event_log_->Log(std::make_unique<RtcEventAudioReceiveStreamConfig>( CreateRtcLogStreamConfig(config))); AudioReceiveStream* receive_stream = new AudioReceiveStream( clock_, &audio_receiver_controller_, transport_send_ptr_->packet_router(), - module_process_thread_.get(), config_.neteq_factory, config, + module_process_thread_->process_thread(), config_.neteq_factory, config, config_.audio_state, event_log_); - { - WriteLockScoped write_lock(*receive_crit_); - receive_rtp_config_.emplace(config.rtp.remote_ssrc, - ReceiveRtpConfig(config)); - audio_receive_streams_.insert(receive_stream); - ConfigureSync(config.sync_group); - } - { - ReadLockScoped read_lock(*send_crit_); - auto it = audio_send_ssrcs_.find(config.rtp.local_ssrc); - if (it != audio_send_ssrcs_.end()) { - receive_stream->AssociateSendStream(it->second); - } + receive_rtp_config_.emplace(config.rtp.remote_ssrc, ReceiveRtpConfig(config)); + audio_receive_streams_.insert(receive_stream); + + ConfigureSync(config.sync_group); + + auto it = audio_send_ssrcs_.find(config.rtp.local_ssrc); + if (it != audio_send_ssrcs_.end()) { + receive_stream->AssociateSendStream(it->second); } + UpdateAggregateNetworkState(); return receive_stream; } @@ -714,26 +852,24 @@ webrtc::AudioReceiveStream* Call::CreateAudioReceiveStream( void Call::DestroyAudioReceiveStream( webrtc::AudioReceiveStream* receive_stream) { TRACE_EVENT0("webrtc", "Call::DestroyAudioReceiveStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RTC_DCHECK(receive_stream != nullptr); webrtc::internal::AudioReceiveStream* audio_receive_stream = static_cast<webrtc::internal::AudioReceiveStream*>(receive_stream); - { - WriteLockScoped write_lock(*receive_crit_); - const AudioReceiveStream::Config& config = audio_receive_stream->config(); - uint32_t ssrc = config.rtp.remote_ssrc; - receive_side_cc_.GetRemoteBitrateEstimator(UseSendSideBwe(config)) - ->RemoveStream(ssrc); - audio_receive_streams_.erase(audio_receive_stream); - const std::string& sync_group = audio_receive_stream->config().sync_group; - const auto it = sync_stream_mapping_.find(sync_group); - if (it != sync_stream_mapping_.end() && - it->second == audio_receive_stream) { - sync_stream_mapping_.erase(it); - ConfigureSync(sync_group); - } - receive_rtp_config_.erase(ssrc); + + const AudioReceiveStream::Config& config = audio_receive_stream->config(); + uint32_t ssrc = config.rtp.remote_ssrc; + receive_side_cc_.GetRemoteBitrateEstimator(UseSendSideBwe(config)) + ->RemoveStream(ssrc); + audio_receive_streams_.erase(audio_receive_stream); + const std::string& sync_group = audio_receive_stream->config().sync_group; + const auto it = sync_stream_mapping_.find(sync_group); + if (it != sync_stream_mapping_.end() && it->second == audio_receive_stream) { + sync_stream_mapping_.erase(it); + ConfigureSync(sync_group); } + receive_rtp_config_.erase(ssrc); + UpdateAggregateNetworkState(); delete audio_receive_stream; } @@ -744,7 +880,7 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( VideoEncoderConfig encoder_config, std::unique_ptr<FecController> fec_controller) { TRACE_EVENT0("webrtc", "Call::CreateVideoSendStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RegisterRateObserver(); @@ -761,20 +897,22 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( std::vector<uint32_t> ssrcs = config.rtp.ssrcs; VideoSendStream* send_stream = new VideoSendStream( - clock_, num_cpu_cores_, module_process_thread_.get(), task_queue_factory_, - call_stats_->AsRtcpRttStats(), transport_send_ptr_, + clock_, num_cpu_cores_, module_process_thread_->process_thread(), + task_queue_factory_, call_stats_->AsRtcpRttStats(), transport_send_ptr_, bitrate_allocator_.get(), video_send_delay_stats_.get(), event_log_, std::move(config), std::move(encoder_config), suspended_video_send_ssrcs_, suspended_video_payload_states_, std::move(fec_controller)); - { - WriteLockScoped write_lock(*send_crit_); - for (uint32_t ssrc : ssrcs) { - RTC_DCHECK(video_send_ssrcs_.find(ssrc) == video_send_ssrcs_.end()); - video_send_ssrcs_[ssrc] = send_stream; - } - video_send_streams_.insert(send_stream); + for (uint32_t ssrc : ssrcs) { + RTC_DCHECK(video_send_ssrcs_.find(ssrc) == video_send_ssrcs_.end()); + video_send_ssrcs_[ssrc] = send_stream; + } + video_send_streams_.insert(send_stream); + // Forward resources that were previously added to the call to the new stream. + for (const auto& resource_forwarder : adaptation_resource_forwarders_) { + resource_forwarder->OnCreateVideoSendStream(send_stream); } + UpdateAggregateNetworkState(); return send_stream; @@ -797,24 +935,27 @@ webrtc::VideoSendStream* Call::CreateVideoSendStream( void Call::DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) { TRACE_EVENT0("webrtc", "Call::DestroyVideoSendStream"); RTC_DCHECK(send_stream != nullptr); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); send_stream->Stop(); VideoSendStream* send_stream_impl = nullptr; - { - WriteLockScoped write_lock(*send_crit_); - auto it = video_send_ssrcs_.begin(); - while (it != video_send_ssrcs_.end()) { - if (it->second == static_cast<VideoSendStream*>(send_stream)) { - send_stream_impl = it->second; - video_send_ssrcs_.erase(it++); - } else { - ++it; - } + + auto it = video_send_ssrcs_.begin(); + while (it != video_send_ssrcs_.end()) { + if (it->second == static_cast<VideoSendStream*>(send_stream)) { + send_stream_impl = it->second; + video_send_ssrcs_.erase(it++); + } else { + ++it; } - video_send_streams_.erase(send_stream_impl); } + // Stop forwarding resources to the stream being destroyed. + for (const auto& resource_forwarder : adaptation_resource_forwarders_) { + resource_forwarder->OnDestroyVideoSendStream(send_stream_impl); + } + video_send_streams_.erase(send_stream_impl); + RTC_CHECK(send_stream_impl != nullptr); VideoSendStream::RtpStateMap rtp_states; @@ -835,7 +976,7 @@ void Call::DestroyVideoSendStream(webrtc::VideoSendStream* send_stream) { webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream( webrtc::VideoReceiveStream::Config configuration) { TRACE_EVENT0("webrtc", "Call::CreateVideoReceiveStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); receive_side_cc_.SetSendPeriodicFeedback( SendPeriodicFeedback(configuration.rtp.extensions)); @@ -847,25 +988,21 @@ webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream( VideoReceiveStream2* receive_stream = new VideoReceiveStream2( task_queue_factory_, current, &video_receiver_controller_, num_cpu_cores_, transport_send_ptr_->packet_router(), std::move(configuration), - module_process_thread_.get(), call_stats_.get(), clock_, + module_process_thread_->process_thread(), call_stats_.get(), clock_, new VCMTiming(clock_)); const webrtc::VideoReceiveStream::Config& config = receive_stream->config(); - { - WriteLockScoped write_lock(*receive_crit_); - if (config.rtp.rtx_ssrc) { - // We record identical config for the rtx stream as for the main - // stream. Since the transport_send_cc negotiation is per payload - // type, we may get an incorrect value for the rtx stream, but - // that is unlikely to matter in practice. - receive_rtp_config_.emplace(config.rtp.rtx_ssrc, - ReceiveRtpConfig(config)); - } - receive_rtp_config_.emplace(config.rtp.remote_ssrc, - ReceiveRtpConfig(config)); - video_receive_streams_.insert(receive_stream); - ConfigureSync(config.sync_group); + if (config.rtp.rtx_ssrc) { + // We record identical config for the rtx stream as for the main + // stream. Since the transport_send_cc negotiation is per payload + // type, we may get an incorrect value for the rtx stream, but + // that is unlikely to matter in practice. + receive_rtp_config_.emplace(config.rtp.rtx_ssrc, ReceiveRtpConfig(config)); } + receive_rtp_config_.emplace(config.rtp.remote_ssrc, ReceiveRtpConfig(config)); + video_receive_streams_.insert(receive_stream); + ConfigureSync(config.sync_group); + receive_stream->SignalNetworkState(video_network_state_); UpdateAggregateNetworkState(); event_log_->Log(std::make_unique<RtcEventVideoReceiveStreamConfig>( @@ -876,22 +1013,20 @@ webrtc::VideoReceiveStream* Call::CreateVideoReceiveStream( void Call::DestroyVideoReceiveStream( webrtc::VideoReceiveStream* receive_stream) { TRACE_EVENT0("webrtc", "Call::DestroyVideoReceiveStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RTC_DCHECK(receive_stream != nullptr); VideoReceiveStream2* receive_stream_impl = static_cast<VideoReceiveStream2*>(receive_stream); const VideoReceiveStream::Config& config = receive_stream_impl->config(); - { - WriteLockScoped write_lock(*receive_crit_); - // Remove all ssrcs pointing to a receive stream. As RTX retransmits on a - // separate SSRC there can be either one or two. - receive_rtp_config_.erase(config.rtp.remote_ssrc); - if (config.rtp.rtx_ssrc) { - receive_rtp_config_.erase(config.rtp.rtx_ssrc); - } - video_receive_streams_.erase(receive_stream_impl); - ConfigureSync(config.sync_group); + + // Remove all ssrcs pointing to a receive stream. As RTX retransmits on a + // separate SSRC there can be either one or two. + receive_rtp_config_.erase(config.rtp.remote_ssrc); + if (config.rtp.rtx_ssrc) { + receive_rtp_config_.erase(config.rtp.rtx_ssrc); } + video_receive_streams_.erase(receive_stream_impl); + ConfigureSync(config.sync_group); receive_side_cc_.GetRemoteBitrateEstimator(UseSendSideBwe(config)) ->RemoveStream(config.rtp.remote_ssrc); @@ -903,30 +1038,25 @@ void Call::DestroyVideoReceiveStream( FlexfecReceiveStream* Call::CreateFlexfecReceiveStream( const FlexfecReceiveStream::Config& config) { TRACE_EVENT0("webrtc", "Call::CreateFlexfecReceiveStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RecoveredPacketReceiver* recovered_packet_receiver = this; FlexfecReceiveStreamImpl* receive_stream; - { - WriteLockScoped write_lock(*receive_crit_); - // Unlike the video and audio receive streams, - // FlexfecReceiveStream implements RtpPacketSinkInterface itself, - // and hence its constructor passes its |this| pointer to - // video_receiver_controller_->CreateStream(). Calling the - // constructor while holding |receive_crit_| ensures that we don't - // call OnRtpPacket until the constructor is finished and the - // object is in a valid state. - // TODO(nisse): Fix constructor so that it can be moved outside of - // this locked scope. - receive_stream = new FlexfecReceiveStreamImpl( - clock_, &video_receiver_controller_, config, recovered_packet_receiver, - call_stats_->AsRtcpRttStats(), module_process_thread_.get()); - - RTC_DCHECK(receive_rtp_config_.find(config.remote_ssrc) == - receive_rtp_config_.end()); - receive_rtp_config_.emplace(config.remote_ssrc, ReceiveRtpConfig(config)); - } + + // Unlike the video and audio receive streams, FlexfecReceiveStream implements + // RtpPacketSinkInterface itself, and hence its constructor passes its |this| + // pointer to video_receiver_controller_->CreateStream(). Calling the + // constructor while on the worker thread ensures that we don't call + // OnRtpPacket until the constructor is finished and the object is + // in a valid state, since OnRtpPacket runs on the same thread. + receive_stream = new FlexfecReceiveStreamImpl( + clock_, &video_receiver_controller_, config, recovered_packet_receiver, + call_stats_->AsRtcpRttStats(), module_process_thread_->process_thread()); + + RTC_DCHECK(receive_rtp_config_.find(config.remote_ssrc) == + receive_rtp_config_.end()); + receive_rtp_config_.emplace(config.remote_ssrc, ReceiveRtpConfig(config)); // TODO(brandtr): Store config in RtcEventLog here. @@ -935,39 +1065,37 @@ FlexfecReceiveStream* Call::CreateFlexfecReceiveStream( void Call::DestroyFlexfecReceiveStream(FlexfecReceiveStream* receive_stream) { TRACE_EVENT0("webrtc", "Call::DestroyFlexfecReceiveStream"); - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); RTC_DCHECK(receive_stream != nullptr); - { - WriteLockScoped write_lock(*receive_crit_); + const FlexfecReceiveStream::Config& config = receive_stream->GetConfig(); + uint32_t ssrc = config.remote_ssrc; + receive_rtp_config_.erase(ssrc); - const FlexfecReceiveStream::Config& config = receive_stream->GetConfig(); - uint32_t ssrc = config.remote_ssrc; - receive_rtp_config_.erase(ssrc); - - // Remove all SSRCs pointing to the FlexfecReceiveStreamImpl to be - // destroyed. - receive_side_cc_.GetRemoteBitrateEstimator(UseSendSideBwe(config)) - ->RemoveStream(ssrc); - } + // Remove all SSRCs pointing to the FlexfecReceiveStreamImpl to be + // destroyed. + receive_side_cc_.GetRemoteBitrateEstimator(UseSendSideBwe(config)) + ->RemoveStream(ssrc); delete receive_stream; } +void Call::AddAdaptationResource(rtc::scoped_refptr<Resource> resource) { + RTC_DCHECK_RUN_ON(worker_thread_); + adaptation_resource_forwarders_.push_back( + std::make_unique<ResourceVideoSendStreamForwarder>(resource)); + const auto& resource_forwarder = adaptation_resource_forwarders_.back(); + for (VideoSendStream* send_stream : video_send_streams_) { + resource_forwarder->OnCreateVideoSendStream(send_stream); + } +} + RtpTransportControllerSendInterface* Call::GetTransportControllerSend() { return transport_send_ptr_; } Call::Stats Call::GetStats() const { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); - - // TODO(tommi): The following stats are managed on the process thread: - // - pacer_delay_ms (PacedSender::Process) - // - rtt_ms - // - recv_bandwidth_bps - // These are delivered on the network TQ: - // - send_bandwidth_bps (see OnTargetTransferRate) - // - max_padding_bitrate_bps (see OnAllocationLimitsChanged) + RTC_DCHECK_RUN_ON(worker_thread_); Stats stats; // TODO(srte): It is unclear if we only want to report queues if network is @@ -983,22 +1111,14 @@ Call::Stats Call::GetStats() const { receive_side_cc_.GetRemoteBitrateEstimator(false)->LatestEstimate( &ssrcs, &recv_bandwidth); stats.recv_bandwidth_bps = recv_bandwidth; - - { - rtc::CritScope cs(&last_bandwidth_bps_crit_); - stats.send_bandwidth_bps = last_bandwidth_bps_; - } - - { - rtc::CritScope cs(&bitrate_crit_); - stats.max_padding_bitrate_bps = configured_max_padding_bitrate_bps_; - } + stats.send_bandwidth_bps = last_bandwidth_bps_; + stats.max_padding_bitrate_bps = configured_max_padding_bitrate_bps_; return stats; } void Call::SignalChannelNetworkState(MediaType media, NetworkState state) { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); switch (media) { case MediaType::AUDIO: audio_network_state_ = state; @@ -1013,40 +1133,25 @@ void Call::SignalChannelNetworkState(MediaType media, NetworkState state) { } UpdateAggregateNetworkState(); - { - ReadLockScoped read_lock(*receive_crit_); - for (VideoReceiveStream2* video_receive_stream : video_receive_streams_) { - video_receive_stream->SignalNetworkState(video_network_state_); - } + for (VideoReceiveStream2* video_receive_stream : video_receive_streams_) { + video_receive_stream->SignalNetworkState(video_network_state_); } } void Call::OnAudioTransportOverheadChanged(int transport_overhead_per_packet) { - ReadLockScoped read_lock(*send_crit_); + RTC_DCHECK_RUN_ON(worker_thread_); for (auto& kv : audio_send_ssrcs_) { kv.second->SetTransportOverhead(transport_overhead_per_packet); } } void Call::UpdateAggregateNetworkState() { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); - bool have_audio = false; - bool have_video = false; - { - ReadLockScoped read_lock(*send_crit_); - if (!audio_send_ssrcs_.empty()) - have_audio = true; - if (!video_send_ssrcs_.empty()) - have_video = true; - } - { - ReadLockScoped read_lock(*receive_crit_); - if (!audio_receive_streams_.empty()) - have_audio = true; - if (!video_receive_streams_.empty()) - have_video = true; - } + bool have_audio = + !audio_send_ssrcs_.empty() || !audio_receive_streams_.empty(); + bool have_video = + !video_send_ssrcs_.empty() || !video_receive_streams_.empty(); bool aggregate_network_up = ((have_video && video_network_state_ == kNetworkUp) || @@ -1073,61 +1178,50 @@ void Call::OnSentPacket(const rtc::SentPacket& sent_packet) { } void Call::OnStartRateUpdate(DataRate start_rate) { - RTC_DCHECK(network_queue()->IsCurrent()); + RTC_DCHECK_RUN_ON(send_transport_queue()); bitrate_allocator_->UpdateStartRate(start_rate.bps<uint32_t>()); } void Call::OnTargetTransferRate(TargetTransferRate msg) { - RTC_DCHECK(network_queue()->IsCurrent()); - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); - { - rtc::CritScope cs(&last_bandwidth_bps_crit_); - last_bandwidth_bps_ = msg.target_rate.bps(); - } + RTC_DCHECK_RUN_ON(send_transport_queue()); uint32_t target_bitrate_bps = msg.target_rate.bps(); // For controlling the rate of feedback messages. receive_side_cc_.OnBitrateChanged(target_bitrate_bps); bitrate_allocator_->OnNetworkEstimateChanged(msg); - // Ignore updates if bitrate is zero (the aggregate network state is down). - if (target_bitrate_bps == 0) { - rtc::CritScope lock(&bitrate_crit_); - estimated_send_bitrate_kbps_counter_.ProcessAndPause(); - pacer_bitrate_kbps_counter_.ProcessAndPause(); - return; - } - - bool sending_video; - { - ReadLockScoped read_lock(*send_crit_); - sending_video = !video_send_streams_.empty(); - } + worker_thread_->PostTask( + ToQueuedTask(task_safety_, [this, target_bitrate_bps]() { + RTC_DCHECK_RUN_ON(worker_thread_); + last_bandwidth_bps_ = target_bitrate_bps; + + // Ignore updates if bitrate is zero (the aggregate network state is + // down) or if we're not sending video. + if (target_bitrate_bps == 0 || video_send_streams_.empty()) { + estimated_send_bitrate_kbps_counter_.ProcessAndPause(); + pacer_bitrate_kbps_counter_.ProcessAndPause(); + return; + } - rtc::CritScope lock(&bitrate_crit_); - if (!sending_video) { - // Do not update the stats if we are not sending video. - estimated_send_bitrate_kbps_counter_.ProcessAndPause(); - pacer_bitrate_kbps_counter_.ProcessAndPause(); - return; - } - estimated_send_bitrate_kbps_counter_.Add(target_bitrate_bps / 1000); - // Pacer bitrate may be higher than bitrate estimate if enforcing min bitrate. - uint32_t pacer_bitrate_bps = - std::max(target_bitrate_bps, min_allocated_send_bitrate_bps_); - pacer_bitrate_kbps_counter_.Add(pacer_bitrate_bps / 1000); + estimated_send_bitrate_kbps_counter_.Add(target_bitrate_bps / 1000); + // Pacer bitrate may be higher than bitrate estimate if enforcing min + // bitrate. + uint32_t pacer_bitrate_bps = + std::max(target_bitrate_bps, min_allocated_send_bitrate_bps_); + pacer_bitrate_kbps_counter_.Add(pacer_bitrate_bps / 1000); + })); } void Call::OnAllocationLimitsChanged(BitrateAllocationLimits limits) { - RTC_DCHECK(network_queue()->IsCurrent()); - RTC_DCHECK_RUN_ON(&worker_sequence_checker_); + RTC_DCHECK_RUN_ON(send_transport_queue()); transport_send_ptr_->SetAllocatedSendBitrateLimits(limits); - min_allocated_send_bitrate_bps_ = limits.min_allocatable_rate.bps(); - - rtc::CritScope lock(&bitrate_crit_); - configured_max_padding_bitrate_bps_ = limits.max_padding_rate.bps(); + worker_thread_->PostTask(ToQueuedTask(task_safety_, [this, limits]() { + RTC_DCHECK_RUN_ON(worker_thread_); + min_allocated_send_bitrate_bps_ = limits.min_allocatable_rate.bps(); + configured_max_padding_bitrate_bps_ = limits.max_padding_rate.bps(); + })); } void Call::ConfigureSync(const std::string& sync_group) { @@ -1194,28 +1288,24 @@ PacketReceiver::DeliveryStatus Call::DeliverRtcp(MediaType media_type, } bool rtcp_delivered = false; if (media_type == MediaType::ANY || media_type == MediaType::VIDEO) { - ReadLockScoped read_lock(*receive_crit_); for (VideoReceiveStream2* stream : video_receive_streams_) { if (stream->DeliverRtcp(packet, length)) rtcp_delivered = true; } } if (media_type == MediaType::ANY || media_type == MediaType::AUDIO) { - ReadLockScoped read_lock(*receive_crit_); for (AudioReceiveStream* stream : audio_receive_streams_) { stream->DeliverRtcp(packet, length); rtcp_delivered = true; } } if (media_type == MediaType::ANY || media_type == MediaType::VIDEO) { - ReadLockScoped read_lock(*send_crit_); for (VideoSendStream* stream : video_send_streams_) { stream->DeliverRtcp(packet, length); rtcp_delivered = true; } } if (media_type == MediaType::ANY || media_type == MediaType::AUDIO) { - ReadLockScoped read_lock(*send_crit_); for (auto& kv : audio_send_ssrcs_) { kv.second->DeliverRtcp(packet, length); rtcp_delivered = true; @@ -1259,17 +1349,15 @@ PacketReceiver::DeliveryStatus Call::DeliverRtp(MediaType media_type, RTC_DCHECK(media_type == MediaType::AUDIO || media_type == MediaType::VIDEO || is_keep_alive_packet); - ReadLockScoped read_lock(*receive_crit_); auto it = receive_rtp_config_.find(parsed_packet.Ssrc()); if (it == receive_rtp_config_.end()) { RTC_LOG(LS_ERROR) << "receive_rtp_config_ lookup failed for ssrc " << parsed_packet.Ssrc(); // Destruction of the receive stream, including deregistering from the - // RtpDemuxer, is not protected by the |receive_crit_| lock. But - // deregistering in the |receive_rtp_config_| map is protected by that lock. - // So by not passing the packet on to demuxing in this case, we prevent - // incoming packets to be passed on via the demuxer to a receive stream - // which is being torned down. + // RtpDemuxer, is not protected by the |worker_thread_|. + // But deregistering in the |receive_rtp_config_| map is. So by not passing + // the packet on to demuxing in this case, we prevent incoming packets to be + // passed on via the demuxer to a receive stream which is being torned down. return DELIVERY_UNKNOWN_SSRC; } @@ -1315,7 +1403,8 @@ PacketReceiver::DeliveryStatus Call::DeliverPacket( MediaType media_type, rtc::CopyOnWriteBuffer packet, int64_t packet_time_us) { - RTC_DCHECK_RUN_ON(&configuration_sequence_checker_); + RTC_DCHECK_RUN_ON(worker_thread_); + if (IsRtcp(packet.cdata(), packet.size())) return DeliverRtcp(media_type, packet.cdata(), packet.size()); @@ -1323,20 +1412,20 @@ PacketReceiver::DeliveryStatus Call::DeliverPacket( } void Call::OnRecoveredPacket(const uint8_t* packet, size_t length) { + RTC_DCHECK_RUN_ON(worker_thread_); RtpPacketReceived parsed_packet; if (!parsed_packet.Parse(packet, length)) return; parsed_packet.set_recovered(true); - ReadLockScoped read_lock(*receive_crit_); auto it = receive_rtp_config_.find(parsed_packet.Ssrc()); if (it == receive_rtp_config_.end()) { RTC_LOG(LS_ERROR) << "receive_rtp_config_ lookup failed for ssrc " << parsed_packet.Ssrc(); // Destruction of the receive stream, including deregistering from the - // RtpDemuxer, is not protected by the |receive_crit_| lock. But - // deregistering in the |receive_rtp_config_| map is protected by that lock. + // RtpDemuxer, is not protected by the |worker_thread_|. + // But deregistering in the |receive_rtp_config_| map is. // So by not passing the packet on to demuxing in this case, we prevent // incoming packets to be passed on via the demuxer to a receive stream // which is being torn down. diff --git a/chromium/third_party/webrtc/call/call.h b/chromium/third_party/webrtc/call/call.h index 77cd3d26901..af9111826ce 100644 --- a/chromium/third_party/webrtc/call/call.h +++ b/chromium/third_party/webrtc/call/call.h @@ -15,6 +15,7 @@ #include <string> #include <vector> +#include "api/adaptation/resource.h" #include "api/media_types.h" #include "call/audio_receive_stream.h" #include "call/audio_send_stream.h" @@ -28,9 +29,46 @@ #include "rtc_base/copy_on_write_buffer.h" #include "rtc_base/network/sent_packet.h" #include "rtc_base/network_route.h" +#include "rtc_base/ref_count.h" namespace webrtc { +// A restricted way to share the module process thread across multiple instances +// of Call that are constructed on the same worker thread (which is what the +// peer connection factory guarantees). +// SharedModuleThread supports a callback that is issued when only one reference +// remains, which is used to indicate to the original owner that the thread may +// be discarded. +class SharedModuleThread : public rtc::RefCountInterface { + protected: + SharedModuleThread(std::unique_ptr<ProcessThread> process_thread, + std::function<void()> on_one_ref_remaining); + friend class rtc::scoped_refptr<SharedModuleThread>; + ~SharedModuleThread() override; + + public: + // Instantiates a default implementation of ProcessThread. + static rtc::scoped_refptr<SharedModuleThread> Create( + const char* name, + std::function<void()> on_one_ref_remaining); + + // Allows injection of an externally created process thread. + static rtc::scoped_refptr<SharedModuleThread> Create( + std::unique_ptr<ProcessThread> process_thread, + std::function<void()> on_one_ref_remaining); + + void EnsureStarted(); + + ProcessThread* process_thread(); + + private: + void AddRef() const override; + rtc::RefCountReleaseStatus Release() const override; + + class Impl; + mutable std::unique_ptr<Impl> impl_; +}; + // A Call instance can contain several send and/or receive streams. All streams // are assumed to have the same remote endpoint and will share bitrate estimates // etc. @@ -50,8 +88,10 @@ class Call { static Call* Create(const Call::Config& config); static Call* Create(const Call::Config& config, + rtc::scoped_refptr<SharedModuleThread> call_thread); + static Call* Create(const Call::Config& config, Clock* clock, - std::unique_ptr<ProcessThread> call_thread, + rtc::scoped_refptr<SharedModuleThread> call_thread, std::unique_ptr<ProcessThread> pacer_thread); virtual AudioSendStream* CreateAudioSendStream( @@ -86,6 +126,11 @@ class Call { virtual void DestroyFlexfecReceiveStream( FlexfecReceiveStream* receive_stream) = 0; + // When a resource is overused, the Call will try to reduce the load on the + // sysem, for example by reducing the resolution or frame rate of encoded + // streams. + virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) = 0; + // All received RTP and RTCP packets for the call should be inserted to this // PacketReceiver. The PacketReceiver pointer is valid as long as the // Call instance exists. diff --git a/chromium/third_party/webrtc/call/call_factory.cc b/chromium/third_party/webrtc/call/call_factory.cc index 6b4f4197425..a3ebc47c6b3 100644 --- a/chromium/third_party/webrtc/call/call_factory.cc +++ b/chromium/third_party/webrtc/call/call_factory.cc @@ -70,7 +70,12 @@ absl::optional<webrtc::BuiltInNetworkBehaviorConfig> ParseDegradationConfig( } } // namespace +CallFactory::CallFactory() { + call_thread_.Detach(); +} + Call* CallFactory::CreateCall(const Call::Config& config) { + RTC_DCHECK_RUN_ON(&call_thread_); absl::optional<webrtc::BuiltInNetworkBehaviorConfig> send_degradation_config = ParseDegradationConfig(true); absl::optional<webrtc::BuiltInNetworkBehaviorConfig> @@ -82,7 +87,14 @@ Call* CallFactory::CreateCall(const Call::Config& config) { config.task_queue_factory); } - return Call::Create(config); + if (!module_thread_) { + module_thread_ = SharedModuleThread::Create("SharedModThread", [this]() { + RTC_DCHECK_RUN_ON(&call_thread_); + module_thread_ = nullptr; + }); + } + + return Call::Create(config, module_thread_); } std::unique_ptr<CallFactoryInterface> CreateCallFactory() { diff --git a/chromium/third_party/webrtc/call/call_factory.h b/chromium/third_party/webrtc/call/call_factory.h index f0d695c915b..65c0b6532ab 100644 --- a/chromium/third_party/webrtc/call/call_factory.h +++ b/chromium/third_party/webrtc/call/call_factory.h @@ -14,13 +14,22 @@ #include "api/call/call_factory_interface.h" #include "call/call.h" #include "call/call_config.h" +#include "rtc_base/synchronization/sequence_checker.h" namespace webrtc { class CallFactory : public CallFactoryInterface { + public: + CallFactory(); + + private: ~CallFactory() override {} Call* CreateCall(const CallConfig& config) override; + + SequenceChecker call_thread_; + rtc::scoped_refptr<SharedModuleThread> module_thread_ + RTC_GUARDED_BY(call_thread_); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/call_unittest.cc b/chromium/third_party/webrtc/call/call_unittest.cc index 8afcf25121b..bd89b873bfa 100644 --- a/chromium/third_party/webrtc/call/call_unittest.cc +++ b/chromium/third_party/webrtc/call/call_unittest.cc @@ -20,13 +20,17 @@ #include "api/rtc_event_log/rtc_event_log.h" #include "api/task_queue/default_task_queue_factory.h" #include "api/test/mock_audio_mixer.h" +#include "api/test/video/function_video_encoder_factory.h" #include "api/transport/field_trial_based_config.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" #include "audio/audio_receive_stream.h" #include "audio/audio_send_stream.h" +#include "call/adaptation/test/fake_resource.h" +#include "call/adaptation/test/mock_resource_listener.h" #include "call/audio_state.h" #include "modules/audio_device/include/mock_audio_device.h" #include "modules/audio_processing/include/mock_audio_processing.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "test/fake_encoder.h" #include "test/gtest.h" #include "test/mock_audio_decoder_factory.h" @@ -35,6 +39,10 @@ namespace { +using ::testing::_; +using ::testing::Contains; +using ::testing::StrictMock; + struct CallHelper { explicit CallHelper(bool use_null_audio_processing) { task_queue_factory_ = webrtc::CreateDefaultTaskQueueFactory(); @@ -67,6 +75,20 @@ struct CallHelper { namespace webrtc { +namespace { + +rtc::scoped_refptr<Resource> FindResourceWhoseNameContains( + const std::vector<rtc::scoped_refptr<Resource>>& resources, + const std::string& name_contains) { + for (const auto& resource : resources) { + if (resource->Name().find(name_contains) != std::string::npos) + return resource; + } + return nullptr; +} + +} // namespace + TEST(CallTest, ConstructDestruct) { for (bool use_null_audio_processing : {false, true}) { CallHelper call(use_null_audio_processing); @@ -325,4 +347,182 @@ TEST(CallTest, RecreatingAudioStreamWithSameSsrcReusesRtpState) { } } +TEST(CallTest, AddAdaptationResourceAfterCreatingVideoSendStream) { + CallHelper call(true); + // Create a VideoSendStream. + test::FunctionVideoEncoderFactory fake_encoder_factory([]() { + return std::make_unique<test::FakeEncoder>(Clock::GetRealTimeClock()); + }); + auto bitrate_allocator_factory = CreateBuiltinVideoBitrateAllocatorFactory(); + MockTransport send_transport; + VideoSendStream::Config config(&send_transport); + config.rtp.payload_type = 110; + config.rtp.ssrcs = {42}; + config.encoder_settings.encoder_factory = &fake_encoder_factory; + config.encoder_settings.bitrate_allocator_factory = + bitrate_allocator_factory.get(); + VideoEncoderConfig encoder_config; + encoder_config.max_bitrate_bps = 1337; + VideoSendStream* stream1 = + call->CreateVideoSendStream(config.Copy(), encoder_config.Copy()); + EXPECT_NE(stream1, nullptr); + config.rtp.ssrcs = {43}; + VideoSendStream* stream2 = + call->CreateVideoSendStream(config.Copy(), encoder_config.Copy()); + EXPECT_NE(stream2, nullptr); + // Add a fake resource. + auto fake_resource = FakeResource::Create("FakeResource"); + call->AddAdaptationResource(fake_resource); + // An adapter resource mirroring the |fake_resource| should now be present on + // both streams. + auto injected_resource1 = FindResourceWhoseNameContains( + stream1->GetAdaptationResources(), fake_resource->Name()); + EXPECT_TRUE(injected_resource1); + auto injected_resource2 = FindResourceWhoseNameContains( + stream2->GetAdaptationResources(), fake_resource->Name()); + EXPECT_TRUE(injected_resource2); + // Overwrite the real resource listeners with mock ones to verify the signal + // gets through. + injected_resource1->SetResourceListener(nullptr); + StrictMock<MockResourceListener> resource_listener1; + EXPECT_CALL(resource_listener1, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([injected_resource1](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(injected_resource1, resource); + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + injected_resource1->SetResourceListener(&resource_listener1); + injected_resource2->SetResourceListener(nullptr); + StrictMock<MockResourceListener> resource_listener2; + EXPECT_CALL(resource_listener2, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([injected_resource2](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(injected_resource2, resource); + EXPECT_EQ(ResourceUsageState::kOveruse, usage_state); + }); + injected_resource2->SetResourceListener(&resource_listener2); + // The kOveruse signal should get to our resource listeners. + fake_resource->SetUsageState(ResourceUsageState::kOveruse); + call->DestroyVideoSendStream(stream1); + call->DestroyVideoSendStream(stream2); +} + +TEST(CallTest, AddAdaptationResourceBeforeCreatingVideoSendStream) { + CallHelper call(true); + // Add a fake resource. + auto fake_resource = FakeResource::Create("FakeResource"); + call->AddAdaptationResource(fake_resource); + // Create a VideoSendStream. + test::FunctionVideoEncoderFactory fake_encoder_factory([]() { + return std::make_unique<test::FakeEncoder>(Clock::GetRealTimeClock()); + }); + auto bitrate_allocator_factory = CreateBuiltinVideoBitrateAllocatorFactory(); + MockTransport send_transport; + VideoSendStream::Config config(&send_transport); + config.rtp.payload_type = 110; + config.rtp.ssrcs = {42}; + config.encoder_settings.encoder_factory = &fake_encoder_factory; + config.encoder_settings.bitrate_allocator_factory = + bitrate_allocator_factory.get(); + VideoEncoderConfig encoder_config; + encoder_config.max_bitrate_bps = 1337; + VideoSendStream* stream1 = + call->CreateVideoSendStream(config.Copy(), encoder_config.Copy()); + EXPECT_NE(stream1, nullptr); + config.rtp.ssrcs = {43}; + VideoSendStream* stream2 = + call->CreateVideoSendStream(config.Copy(), encoder_config.Copy()); + EXPECT_NE(stream2, nullptr); + // An adapter resource mirroring the |fake_resource| should be present on both + // streams. + auto injected_resource1 = FindResourceWhoseNameContains( + stream1->GetAdaptationResources(), fake_resource->Name()); + EXPECT_TRUE(injected_resource1); + auto injected_resource2 = FindResourceWhoseNameContains( + stream2->GetAdaptationResources(), fake_resource->Name()); + EXPECT_TRUE(injected_resource2); + // Overwrite the real resource listeners with mock ones to verify the signal + // gets through. + injected_resource1->SetResourceListener(nullptr); + StrictMock<MockResourceListener> resource_listener1; + EXPECT_CALL(resource_listener1, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([injected_resource1](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(injected_resource1, resource); + EXPECT_EQ(ResourceUsageState::kUnderuse, usage_state); + }); + injected_resource1->SetResourceListener(&resource_listener1); + injected_resource2->SetResourceListener(nullptr); + StrictMock<MockResourceListener> resource_listener2; + EXPECT_CALL(resource_listener2, OnResourceUsageStateMeasured(_, _)) + .Times(1) + .WillOnce([injected_resource2](rtc::scoped_refptr<Resource> resource, + ResourceUsageState usage_state) { + EXPECT_EQ(injected_resource2, resource); + EXPECT_EQ(ResourceUsageState::kUnderuse, usage_state); + }); + injected_resource2->SetResourceListener(&resource_listener2); + // The kUnderuse signal should get to our resource listeners. + fake_resource->SetUsageState(ResourceUsageState::kUnderuse); + call->DestroyVideoSendStream(stream1); + call->DestroyVideoSendStream(stream2); +} + +TEST(CallTest, SharedModuleThread) { + class SharedModuleThreadUser : public Module { + public: + SharedModuleThreadUser(ProcessThread* expected_thread, + rtc::scoped_refptr<SharedModuleThread> thread) + : expected_thread_(expected_thread), thread_(std::move(thread)) { + thread_->EnsureStarted(); + thread_->process_thread()->RegisterModule(this, RTC_FROM_HERE); + } + + ~SharedModuleThreadUser() override { + thread_->process_thread()->DeRegisterModule(this); + EXPECT_TRUE(thread_was_checked_); + } + + private: + int64_t TimeUntilNextProcess() override { return 1000; } + void Process() override {} + void ProcessThreadAttached(ProcessThread* process_thread) override { + if (!process_thread) { + // Being detached. + return; + } + EXPECT_EQ(process_thread, expected_thread_); + thread_was_checked_ = true; + } + + bool thread_was_checked_ = false; + ProcessThread* const expected_thread_; + rtc::scoped_refptr<SharedModuleThread> thread_; + }; + + // Create our test instance and pass a lambda to it that gets executed when + // the reference count goes back to 1 - meaning |shared| again is the only + // reference, which means we can free the variable and deallocate the thread. + rtc::scoped_refptr<SharedModuleThread> shared; + shared = SharedModuleThread::Create("MySharedProcessThread", + [&shared]() { shared = nullptr; }); + ProcessThread* process_thread = shared->process_thread(); + + ASSERT_TRUE(shared.get()); + + { + // Create a couple of users of the thread. + // These instances are in a separate scope to trigger the callback to our + // lambda, which will run when these go out of scope. + SharedModuleThreadUser user1(process_thread, shared); + SharedModuleThreadUser user2(process_thread, shared); + } + + // The thread should now have been stopped and freed. + EXPECT_FALSE(shared); +} + } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/degraded_call.cc b/chromium/third_party/webrtc/call/degraded_call.cc index 9c8d2be5081..007e0af3602 100644 --- a/chromium/third_party/webrtc/call/degraded_call.cc +++ b/chromium/third_party/webrtc/call/degraded_call.cc @@ -245,6 +245,11 @@ void DegradedCall::DestroyFlexfecReceiveStream( call_->DestroyFlexfecReceiveStream(receive_stream); } +void DegradedCall::AddAdaptationResource( + rtc::scoped_refptr<Resource> resource) { + call_->AddAdaptationResource(std::move(resource)); +} + PacketReceiver* DegradedCall::Receiver() { if (receive_config_) { return this; diff --git a/chromium/third_party/webrtc/call/degraded_call.h b/chromium/third_party/webrtc/call/degraded_call.h index 49230ca1edc..ac072b71594 100644 --- a/chromium/third_party/webrtc/call/degraded_call.h +++ b/chromium/third_party/webrtc/call/degraded_call.h @@ -77,6 +77,8 @@ class DegradedCall : public Call, private PacketReceiver { void DestroyFlexfecReceiveStream( FlexfecReceiveStream* receive_stream) override; + void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) override; + PacketReceiver* Receiver() override; RtpTransportControllerSendInterface* GetTransportControllerSend() override; diff --git a/chromium/third_party/webrtc/call/fake_network_pipe_unittest.cc b/chromium/third_party/webrtc/call/fake_network_pipe_unittest.cc index 9c4a3bf7558..852a4272220 100644 --- a/chromium/third_party/webrtc/call/fake_network_pipe_unittest.cc +++ b/chromium/third_party/webrtc/call/fake_network_pipe_unittest.cc @@ -24,8 +24,10 @@ namespace webrtc { class MockReceiver : public PacketReceiver { public: - MOCK_METHOD3(DeliverPacket, - DeliveryStatus(MediaType, rtc::CopyOnWriteBuffer, int64_t)); + MOCK_METHOD(DeliveryStatus, + DeliverPacket, + (MediaType, rtc::CopyOnWriteBuffer, int64_t), + (override)); virtual ~MockReceiver() = default; }; diff --git a/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.cc b/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.cc index 40005efe835..e629bca3477 100644 --- a/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.cc +++ b/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.cc @@ -22,7 +22,6 @@ #include "call/rtp_stream_receiver_controller_interface.h" #include "modules/rtp_rtcp/include/flexfec_receiver.h" #include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/utility/include/process_thread.h" #include "rtc_base/checks.h" @@ -119,12 +118,12 @@ std::unique_ptr<FlexfecReceiver> MaybeCreateFlexfecReceiver( recovered_packet_receiver)); } -std::unique_ptr<RtpRtcp> CreateRtpRtcpModule( +std::unique_ptr<ModuleRtpRtcpImpl2> CreateRtpRtcpModule( Clock* clock, ReceiveStatistics* receive_statistics, const FlexfecReceiveStreamImpl::Config& config, RtcpRttStats* rtt_stats) { - RtpRtcp::Configuration configuration; + RtpRtcpInterface::Configuration configuration; configuration.audio = false; configuration.receiver_only = true; configuration.clock = clock; @@ -132,7 +131,7 @@ std::unique_ptr<RtpRtcp> CreateRtpRtcpModule( configuration.outgoing_transport = config.rtcp_send_transport; configuration.rtt_stats = rtt_stats; configuration.local_media_ssrc = config.local_ssrc; - return RtpRtcp::Create(configuration); + return ModuleRtpRtcpImpl2::Create(configuration); } } // namespace diff --git a/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.h b/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.h index d4fdc7431a7..888dae9ebd5 100644 --- a/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.h +++ b/chromium/third_party/webrtc/call/flexfec_receive_stream_impl.h @@ -15,6 +15,7 @@ #include "call/flexfec_receive_stream.h" #include "call/rtp_packet_sink_interface.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "system_wrappers/include/clock.h" namespace webrtc { @@ -55,7 +56,7 @@ class FlexfecReceiveStreamImpl : public FlexfecReceiveStream { // RTCP reporting. const std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_; - const std::unique_ptr<RtpRtcp> rtp_rtcp_; + const std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_; ProcessThread* process_thread_; std::unique_ptr<RtpStreamReceiverInterface> rtp_stream_receiver_; diff --git a/chromium/third_party/webrtc/call/rampup_tests.cc b/chromium/third_party/webrtc/call/rampup_tests.cc index 64eab050cbe..89fbe3dde7c 100644 --- a/chromium/third_party/webrtc/call/rampup_tests.cc +++ b/chromium/third_party/webrtc/call/rampup_tests.cc @@ -362,14 +362,14 @@ void RampUpTester::AccumulateStats(const VideoSendStream::StreamStats& stream, void RampUpTester::TriggerTestDone() { RTC_DCHECK_GE(test_start_ms_, 0); - // TODO(holmer): Add audio send stats here too when those APIs are available. - if (!send_stream_) - return; - // Stop polling stats. // Corner case for field_trials=WebRTC-QuickPerfTest/Enabled/ SendTask(RTC_FROM_HERE, task_queue_, [this] { pending_task_.Stop(); }); + // TODO(holmer): Add audio send stats here too when those APIs are available. + if (!send_stream_) + return; + VideoSendStream::Stats send_stats = send_stream_->GetStats(); send_stream_ = nullptr; // To avoid dereferencing a bad pointer. diff --git a/chromium/third_party/webrtc/call/rtcp_demuxer_unittest.cc b/chromium/third_party/webrtc/call/rtcp_demuxer_unittest.cc index 23c305c9007..f3949ca78b4 100644 --- a/chromium/third_party/webrtc/call/rtcp_demuxer_unittest.cc +++ b/chromium/third_party/webrtc/call/rtcp_demuxer_unittest.cc @@ -37,7 +37,7 @@ using ::testing::NiceMock; class MockRtcpPacketSink : public RtcpPacketSinkInterface { public: - MOCK_METHOD1(OnRtcpPacket, void(rtc::ArrayView<const uint8_t>)); + MOCK_METHOD(void, OnRtcpPacket, (rtc::ArrayView<const uint8_t>), (override)); }; class RtcpDemuxerTest : public ::testing::Test { @@ -81,6 +81,8 @@ class RtcpDemuxerTest : public ::testing::Test { std::set<RtcpPacketSinkInterface*> broadcast_sinks_to_tear_down_; }; +class RtcpDemuxerDeathTest : public RtcpDemuxerTest {}; + // Produces a packet buffer representing an RTCP packet with a given SSRC, // as it would look when sent over the wire. // |distinguishing_string| allows different RTCP packets with the same SSRC @@ -419,7 +421,7 @@ TEST_F(RtcpDemuxerTest, FirstResolutionOfRsidNotForgotten) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST_F(RtcpDemuxerTest, RepeatedSsrcToSinkAssociationsDisallowed) { +TEST_F(RtcpDemuxerDeathTest, RepeatedSsrcToSinkAssociationsDisallowed) { MockRtcpPacketSink sink; constexpr uint32_t ssrc = 101; @@ -427,7 +429,7 @@ TEST_F(RtcpDemuxerTest, RepeatedSsrcToSinkAssociationsDisallowed) { EXPECT_DEATH(AddSsrcSink(ssrc, &sink), ""); } -TEST_F(RtcpDemuxerTest, RepeatedRsidToSinkAssociationsDisallowed) { +TEST_F(RtcpDemuxerDeathTest, RepeatedRsidToSinkAssociationsDisallowed) { MockRtcpPacketSink sink; const std::string rsid = "z"; @@ -435,14 +437,14 @@ TEST_F(RtcpDemuxerTest, RepeatedRsidToSinkAssociationsDisallowed) { EXPECT_DEATH(AddRsidSink(rsid, &sink), ""); } -TEST_F(RtcpDemuxerTest, RepeatedBroadcastSinkRegistrationDisallowed) { +TEST_F(RtcpDemuxerDeathTest, RepeatedBroadcastSinkRegistrationDisallowed) { MockRtcpPacketSink sink; AddBroadcastSink(&sink); EXPECT_DEATH(AddBroadcastSink(&sink), ""); } -TEST_F(RtcpDemuxerTest, SsrcSinkCannotAlsoBeRegisteredAsBroadcast) { +TEST_F(RtcpDemuxerDeathTest, SsrcSinkCannotAlsoBeRegisteredAsBroadcast) { MockRtcpPacketSink sink; constexpr uint32_t ssrc = 101; @@ -450,7 +452,7 @@ TEST_F(RtcpDemuxerTest, SsrcSinkCannotAlsoBeRegisteredAsBroadcast) { EXPECT_DEATH(AddBroadcastSink(&sink), ""); } -TEST_F(RtcpDemuxerTest, RsidSinkCannotAlsoBeRegisteredAsBroadcast) { +TEST_F(RtcpDemuxerDeathTest, RsidSinkCannotAlsoBeRegisteredAsBroadcast) { MockRtcpPacketSink sink; const std::string rsid = "z"; @@ -458,7 +460,7 @@ TEST_F(RtcpDemuxerTest, RsidSinkCannotAlsoBeRegisteredAsBroadcast) { EXPECT_DEATH(AddBroadcastSink(&sink), ""); } -TEST_F(RtcpDemuxerTest, BroadcastSinkCannotAlsoBeRegisteredAsSsrcSink) { +TEST_F(RtcpDemuxerDeathTest, BroadcastSinkCannotAlsoBeRegisteredAsSsrcSink) { MockRtcpPacketSink sink; AddBroadcastSink(&sink); @@ -466,7 +468,7 @@ TEST_F(RtcpDemuxerTest, BroadcastSinkCannotAlsoBeRegisteredAsSsrcSink) { EXPECT_DEATH(AddSsrcSink(ssrc, &sink), ""); } -TEST_F(RtcpDemuxerTest, BroadcastSinkCannotAlsoBeRegisteredAsRsidSink) { +TEST_F(RtcpDemuxerDeathTest, BroadcastSinkCannotAlsoBeRegisteredAsRsidSink) { MockRtcpPacketSink sink; AddBroadcastSink(&sink); @@ -474,27 +476,27 @@ TEST_F(RtcpDemuxerTest, BroadcastSinkCannotAlsoBeRegisteredAsRsidSink) { EXPECT_DEATH(AddRsidSink(rsid, &sink), ""); } -TEST_F(RtcpDemuxerTest, MayNotCallRemoveSinkOnNeverAddedSink) { +TEST_F(RtcpDemuxerDeathTest, MayNotCallRemoveSinkOnNeverAddedSink) { MockRtcpPacketSink sink; EXPECT_DEATH(RemoveSink(&sink), ""); } -TEST_F(RtcpDemuxerTest, MayNotCallRemoveBroadcastSinkOnNeverAddedSink) { +TEST_F(RtcpDemuxerDeathTest, MayNotCallRemoveBroadcastSinkOnNeverAddedSink) { MockRtcpPacketSink sink; EXPECT_DEATH(RemoveBroadcastSink(&sink), ""); } -TEST_F(RtcpDemuxerTest, RsidMustBeNonEmpty) { +TEST_F(RtcpDemuxerDeathTest, RsidMustBeNonEmpty) { MockRtcpPacketSink sink; EXPECT_DEATH(AddRsidSink("", &sink), ""); } -TEST_F(RtcpDemuxerTest, RsidMustBeAlphaNumeric) { +TEST_F(RtcpDemuxerDeathTest, RsidMustBeAlphaNumeric) { MockRtcpPacketSink sink; EXPECT_DEATH(AddRsidSink("a_3", &sink), ""); } -TEST_F(RtcpDemuxerTest, RsidMustNotExceedMaximumLength) { +TEST_F(RtcpDemuxerDeathTest, RsidMustNotExceedMaximumLength) { MockRtcpPacketSink sink; std::string rsid(BaseRtpStringExtension::kMaxValueSizeBytes + 1, 'a'); EXPECT_DEATH(AddRsidSink(rsid, &sink), ""); diff --git a/chromium/third_party/webrtc/call/rtp_demuxer.cc b/chromium/third_party/webrtc/call/rtp_demuxer.cc index 14725cf023d..3ab75c7f982 100644 --- a/chromium/third_party/webrtc/call/rtp_demuxer.cc +++ b/chromium/third_party/webrtc/call/rtp_demuxer.cc @@ -24,6 +24,25 @@ namespace webrtc { RtpDemuxerCriteria::RtpDemuxerCriteria() = default; RtpDemuxerCriteria::~RtpDemuxerCriteria() = default; +std::string RtpDemuxerCriteria::ToString() const { + rtc::StringBuilder sb; + sb << "{mid: " << (mid.empty() ? "<empty>" : mid) + << ", rsid: " << (rsid.empty() ? "<empty>" : rsid) << ", ssrcs: ["; + + for (auto ssrc : ssrcs) { + sb << ssrc << ", "; + } + + sb << "], payload_types = ["; + + for (auto pt : payload_types) { + sb << pt << ", "; + } + + sb << "]}"; + return sb.Release(); +} + // static std::string RtpDemuxer::DescribePacket(const RtpPacketReceived& packet) { rtc::StringBuilder sb; @@ -66,6 +85,8 @@ bool RtpDemuxer::AddSink(const RtpDemuxerCriteria& criteria, // criteria because new sinks are created according to user-specified SDP and // we do not want to crash due to a data validation error. if (CriteriaWouldConflict(criteria)) { + RTC_LOG(LS_ERROR) << "Unable to add sink = " << sink + << " due conflicting criteria " << criteria.ToString(); return false; } @@ -92,6 +113,9 @@ bool RtpDemuxer::AddSink(const RtpDemuxerCriteria& criteria, RefreshKnownMids(); + RTC_LOG(LS_INFO) << "Added sink = " << sink << " for criteria " + << criteria.ToString(); + return true; } @@ -105,25 +129,40 @@ bool RtpDemuxer::CriteriaWouldConflict( // Adding this criteria would cause one of these rules to be shadowed, so // reject this new criteria. if (known_mids_.find(criteria.mid) != known_mids_.end()) { + RTC_LOG(LS_INFO) << criteria.ToString() + << " would conflict with known mid"; return true; } } else { // If the exact rule already exists, then reject this duplicate. - if (sink_by_mid_and_rsid_.find(std::make_pair( - criteria.mid, criteria.rsid)) != sink_by_mid_and_rsid_.end()) { + const auto sink_by_mid_and_rsid = sink_by_mid_and_rsid_.find( + std::make_pair(criteria.mid, criteria.rsid)); + if (sink_by_mid_and_rsid != sink_by_mid_and_rsid_.end()) { + RTC_LOG(LS_INFO) << criteria.ToString() + << " would conflict with existing sink = " + << sink_by_mid_and_rsid->second + << " by mid+rsid binding"; return true; } // If there is already a sink registered for the bare MID, then this // criteria will never receive any packets because they will just be // directed to that MID sink, so reject this new criteria. - if (sink_by_mid_.find(criteria.mid) != sink_by_mid_.end()) { + const auto sink_by_mid = sink_by_mid_.find(criteria.mid); + if (sink_by_mid != sink_by_mid_.end()) { + RTC_LOG(LS_INFO) << criteria.ToString() + << " would conflict with existing sink = " + << sink_by_mid->second << " by mid binding"; return true; } } } for (uint32_t ssrc : criteria.ssrcs) { - if (sink_by_ssrc_.find(ssrc) != sink_by_ssrc_.end()) { + const auto sink_by_ssrc = sink_by_ssrc_.find(ssrc); + if (sink_by_ssrc != sink_by_ssrc_.end()) { + RTC_LOG(LS_INFO) << criteria.ToString() + << " would conflict with existing sink = " + << sink_by_ssrc->second << " binding by SSRC=" << ssrc; return true; } } @@ -168,7 +207,11 @@ bool RtpDemuxer::RemoveSink(const RtpPacketSinkInterface* sink) { RemoveFromMapByValue(&sink_by_mid_and_rsid_, sink) + RemoveFromMapByValue(&sink_by_rsid_, sink); RefreshKnownMids(); - return num_removed > 0; + bool removed = num_removed > 0; + if (removed) { + RTC_LOG(LS_INFO) << "Removed sink = " << sink << " bindings"; + } + return removed; } bool RtpDemuxer::OnRtpPacket(const RtpPacketReceived& packet) { @@ -370,9 +413,13 @@ bool RtpDemuxer::AddSsrcSinkBinding(uint32_t ssrc, auto it = result.first; bool inserted = result.second; if (inserted) { + RTC_LOG(LS_INFO) << "Added sink = " << sink + << " binding with SSRC=" << ssrc; return true; } if (it->second != sink) { + RTC_LOG(LS_INFO) << "Updated sink = " << sink + << " binding with SSRC=" << ssrc; it->second = sink; return true; } diff --git a/chromium/third_party/webrtc/call/rtp_demuxer.h b/chromium/third_party/webrtc/call/rtp_demuxer.h index c815c47f724..dae7a53b702 100644 --- a/chromium/third_party/webrtc/call/rtp_demuxer.h +++ b/chromium/third_party/webrtc/call/rtp_demuxer.h @@ -44,6 +44,9 @@ struct RtpDemuxerCriteria { // Will match packets with any of these payload types. std::set<uint8_t> payload_types; + + // Return string representation of demux criteria to facilitate logging + std::string ToString() const; }; // This class represents the RTP demuxing, for a single RTP session (i.e., one diff --git a/chromium/third_party/webrtc/call/rtp_demuxer_unittest.cc b/chromium/third_party/webrtc/call/rtp_demuxer_unittest.cc index 86b458a0cc0..59baafe9ff8 100644 --- a/chromium/third_party/webrtc/call/rtp_demuxer_unittest.cc +++ b/chromium/third_party/webrtc/call/rtp_demuxer_unittest.cc @@ -37,14 +37,22 @@ using ::testing::NiceMock; class MockSsrcBindingObserver : public SsrcBindingObserver { public: - MOCK_METHOD2(OnSsrcBoundToRsid, void(const std::string& rsid, uint32_t ssrc)); - MOCK_METHOD2(OnSsrcBoundToMid, void(const std::string& mid, uint32_t ssrc)); - MOCK_METHOD3(OnSsrcBoundToMidRsid, - void(const std::string& mid, - const std::string& rsid, - uint32_t ssrc)); - MOCK_METHOD2(OnSsrcBoundToPayloadType, - void(uint8_t payload_type, uint32_t ssrc)); + MOCK_METHOD(void, + OnSsrcBoundToRsid, + (const std::string& rsid, uint32_t ssrc), + (override)); + MOCK_METHOD(void, + OnSsrcBoundToMid, + (const std::string& mid, uint32_t ssrc), + (override)); + MOCK_METHOD(void, + OnSsrcBoundToMidRsid, + (const std::string& mid, const std::string& rsid, uint32_t ssrc), + (override)); + MOCK_METHOD(void, + OnSsrcBoundToPayloadType, + (uint8_t payload_type, uint32_t ssrc), + (override)); }; class RtpDemuxerTest : public ::testing::Test { @@ -210,6 +218,8 @@ class RtpDemuxerTest : public ::testing::Test { uint16_t next_sequence_number_ = 1; }; +class RtpDemuxerDeathTest : public RtpDemuxerTest {}; + MATCHER_P(SamePacketAs, other, "") { return arg.Ssrc() == other.Ssrc() && arg.SequenceNumber() == other.SequenceNumber(); @@ -1478,41 +1488,42 @@ TEST_F(RtpDemuxerTest, MaliciousPeerCannotCauseMemoryOveruse) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST_F(RtpDemuxerTest, CriteriaMustBeNonEmpty) { +TEST_F(RtpDemuxerDeathTest, CriteriaMustBeNonEmpty) { MockRtpPacketSink sink; RtpDemuxerCriteria criteria; EXPECT_DEATH(AddSink(criteria, &sink), ""); } -TEST_F(RtpDemuxerTest, RsidMustBeAlphaNumeric) { +TEST_F(RtpDemuxerDeathTest, RsidMustBeAlphaNumeric) { MockRtpPacketSink sink; EXPECT_DEATH(AddSinkOnlyRsid("a_3", &sink), ""); } -TEST_F(RtpDemuxerTest, MidMustBeToken) { +TEST_F(RtpDemuxerDeathTest, MidMustBeToken) { MockRtpPacketSink sink; EXPECT_DEATH(AddSinkOnlyMid("a(3)", &sink), ""); } -TEST_F(RtpDemuxerTest, RsidMustNotExceedMaximumLength) { +TEST_F(RtpDemuxerDeathTest, RsidMustNotExceedMaximumLength) { MockRtpPacketSink sink; std::string rsid(BaseRtpStringExtension::kMaxValueSizeBytes + 1, 'a'); EXPECT_DEATH(AddSinkOnlyRsid(rsid, &sink), ""); } -TEST_F(RtpDemuxerTest, MidMustNotExceedMaximumLength) { +TEST_F(RtpDemuxerDeathTest, MidMustNotExceedMaximumLength) { MockRtpPacketSink sink; std::string mid(BaseRtpStringExtension::kMaxValueSizeBytes + 1, 'a'); EXPECT_DEATH(AddSinkOnlyMid(mid, &sink), ""); } -TEST_F(RtpDemuxerTest, DoubleRegisterationOfSsrcBindingObserverDisallowed) { +TEST_F(RtpDemuxerDeathTest, + DoubleRegisterationOfSsrcBindingObserverDisallowed) { MockSsrcBindingObserver observer; RegisterSsrcBindingObserver(&observer); EXPECT_DEATH(RegisterSsrcBindingObserver(&observer), ""); } -TEST_F(RtpDemuxerTest, +TEST_F(RtpDemuxerDeathTest, DregisterationOfNeverRegisteredSsrcBindingObserverDisallowed) { MockSsrcBindingObserver observer; EXPECT_DEATH(DeregisterSsrcBindingObserver(&observer), ""); diff --git a/chromium/third_party/webrtc/call/rtp_payload_params.cc b/chromium/third_party/webrtc/call/rtp_payload_params.cc index 279eb588d73..110db2e9fab 100644 --- a/chromium/third_party/webrtc/call/rtp_payload_params.cc +++ b/chromium/third_party/webrtc/call/rtp_payload_params.cc @@ -93,15 +93,6 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info, h264_header.packetization_mode = info.codecSpecific.H264.packetization_mode; rtp->simulcastIdx = spatial_index.value_or(0); - rtp->frame_marking.temporal_id = kNoTemporalIdx; - if (info.codecSpecific.H264.temporal_idx != kNoTemporalIdx) { - rtp->frame_marking.temporal_id = info.codecSpecific.H264.temporal_idx; - rtp->frame_marking.layer_id = 0; - rtp->frame_marking.independent_frame = - info.codecSpecific.H264.idr_frame; - rtp->frame_marking.base_layer_sync = - info.codecSpecific.H264.base_layer_sync; - } return; } case kVideoCodecMultiplex: @@ -139,10 +130,7 @@ RtpPayloadParams::RtpPayloadParams(const uint32_t ssrc, : ssrc_(ssrc), generic_picture_id_experiment_( absl::StartsWith(trials.Lookup("WebRTC-GenericPictureId"), - "Enabled")), - generic_descriptor_experiment_( - !absl::StartsWith(trials.Lookup("WebRTC-GenericDescriptor"), - "Disabled")) { + "Enabled")) { for (auto& spatial_layer : last_shared_frame_id_) spatial_layer.fill(-1); @@ -186,9 +174,8 @@ RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader( SetCodecSpecific(&rtp_video_header, first_frame_in_picture); - if (generic_descriptor_experiment_) - SetGeneric(codec_specific_info, shared_frame_id, is_keyframe, - &rtp_video_header); + SetGeneric(codec_specific_info, shared_frame_id, is_keyframe, + &rtp_video_header); return rtp_video_header; } @@ -237,14 +224,6 @@ void RtpPayloadParams::SetCodecSpecific(RTPVideoHeader* rtp_video_header, vp9_header.tl0_pic_idx = state_.tl0_pic_idx; } } - if (rtp_video_header->codec == kVideoCodecH264) { - if (rtp_video_header->frame_marking.temporal_id != kNoTemporalIdx) { - if (rtp_video_header->frame_marking.temporal_id == 0) { - ++state_.tl0_pic_idx; - } - rtp_video_header->frame_marking.tl0_pic_idx = state_.tl0_pic_idx; - } - } if (generic_picture_id_experiment_ && rtp_video_header->codec == kVideoCodecGeneric) { rtp_video_header->video_type_header.emplace<RTPVideoHeaderLegacyGeneric>() @@ -261,6 +240,8 @@ RtpPayloadParams::GenericDescriptorFromFrameInfo( generic.frame_id = frame_id; generic.dependencies = dependencies_calculator_.FromBuffersUsage( frame_type, frame_id, frame_info.encoder_buffers); + generic.chain_diffs = + chains_calculator_.From(frame_id, frame_info.part_of_chain); generic.spatial_index = frame_info.spatial_id; generic.temporal_index = frame_info.temporal_id; generic.decode_target_indications = frame_info.decode_target_indications; @@ -273,6 +254,11 @@ void RtpPayloadParams::SetGeneric(const CodecSpecificInfo* codec_specific_info, RTPVideoHeader* rtp_video_header) { if (codec_specific_info && codec_specific_info->generic_frame_info && !codec_specific_info->generic_frame_info->encoder_buffers.empty()) { + if (is_keyframe) { + // Key frame resets all chains it is in. + chains_calculator_.Reset( + codec_specific_info->generic_frame_info->part_of_chain); + } rtp_video_header->generic = GenericDescriptorFromFrameInfo(*codec_specific_info->generic_frame_info, frame_id, rtp_video_header->frame_type); diff --git a/chromium/third_party/webrtc/call/rtp_payload_params.h b/chromium/third_party/webrtc/call/rtp_payload_params.h index 13b10503781..2e0faeb5c99 100644 --- a/chromium/third_party/webrtc/call/rtp_payload_params.h +++ b/chromium/third_party/webrtc/call/rtp_payload_params.h @@ -19,6 +19,7 @@ #include "call/rtp_config.h" #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/video_coding/chain_diff_calculator.h" #include "modules/video_coding/frame_dependencies_calculator.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -88,6 +89,7 @@ class RtpPayloadParams final { RTPVideoHeader::GenericDescriptorInfo* generic); FrameDependenciesCalculator dependencies_calculator_; + ChainDiffCalculator chains_calculator_; // TODO(bugs.webrtc.org/10242): Remove once all encoder-wrappers are updated. // Holds the last shared frame id for a given (spatial, temporal) layer. std::array<std::array<int64_t, RtpGenericFrameDescriptor::kMaxTemporalLayers>, @@ -112,7 +114,6 @@ class RtpPayloadParams final { RtpPayloadState state_; const bool generic_picture_id_experiment_; - const bool generic_descriptor_experiment_; }; } // namespace webrtc #endif // CALL_RTP_PAYLOAD_PARAMS_H_ diff --git a/chromium/third_party/webrtc/call/rtp_payload_params_unittest.cc b/chromium/third_party/webrtc/call/rtp_payload_params_unittest.cc index 1045504b44e..a5510b0240d 100644 --- a/chromium/third_party/webrtc/call/rtp_payload_params_unittest.cc +++ b/chromium/third_party/webrtc/call/rtp_payload_params_unittest.cc @@ -32,6 +32,7 @@ using ::testing::ElementsAre; using ::testing::IsEmpty; +using ::testing::SizeIs; namespace webrtc { namespace { @@ -147,54 +148,6 @@ TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_Vp9) { codec_info.codecSpecific.VP9.end_of_picture); } -TEST(RtpPayloadParamsTest, InfoMappedToRtpVideoHeader_H264) { - RtpPayloadState state; - state.picture_id = kPictureId; - state.tl0_pic_idx = kInitialTl0PicIdx1; - RtpPayloadParams params(kSsrc1, &state, FieldTrialBasedConfig()); - - EncodedImage encoded_image; - CodecSpecificInfo codec_info; - CodecSpecificInfoH264* h264info = &codec_info.codecSpecific.H264; - codec_info.codecType = kVideoCodecH264; - h264info->packetization_mode = H264PacketizationMode::SingleNalUnit; - h264info->temporal_idx = kNoTemporalIdx; - - RTPVideoHeader header = - params.GetRtpVideoHeader(encoded_image, &codec_info, 10); - - EXPECT_EQ(0, header.simulcastIdx); - EXPECT_EQ(kVideoCodecH264, header.codec); - const auto& h264 = absl::get<RTPVideoHeaderH264>(header.video_type_header); - EXPECT_EQ(H264PacketizationMode::SingleNalUnit, h264.packetization_mode); - - // test temporal param 1 - h264info->temporal_idx = 1; - h264info->base_layer_sync = true; - h264info->idr_frame = false; - - header = params.GetRtpVideoHeader(encoded_image, &codec_info, 20); - - EXPECT_EQ(kVideoCodecH264, header.codec); - EXPECT_EQ(header.frame_marking.tl0_pic_idx, kInitialTl0PicIdx1); - EXPECT_EQ(header.frame_marking.temporal_id, h264info->temporal_idx); - EXPECT_EQ(header.frame_marking.base_layer_sync, h264info->base_layer_sync); - EXPECT_EQ(header.frame_marking.independent_frame, h264info->idr_frame); - - // test temporal param 2 - h264info->temporal_idx = 0; - h264info->base_layer_sync = false; - h264info->idr_frame = true; - - header = params.GetRtpVideoHeader(encoded_image, &codec_info, 30); - - EXPECT_EQ(kVideoCodecH264, header.codec); - EXPECT_EQ(header.frame_marking.tl0_pic_idx, kInitialTl0PicIdx1 + 1); - EXPECT_EQ(header.frame_marking.temporal_id, h264info->temporal_idx); - EXPECT_EQ(header.frame_marking.base_layer_sync, h264info->base_layer_sync); - EXPECT_EQ(header.frame_marking.independent_frame, h264info->idr_frame); -} - TEST(RtpPayloadParamsTest, PictureIdIsSetForVp8) { RtpPayloadState state; state.picture_id = kInitialPictureId1; @@ -349,8 +302,6 @@ TEST(RtpPayloadParamsTest, PictureIdForOldGenericFormat) { } TEST(RtpPayloadParamsTest, GenericDescriptorForGenericCodec) { - test::ScopedFieldTrials generic_picture_id( - "WebRTC-GenericDescriptor/Enabled/"); RtpPayloadState state{}; EncodedImage encoded_image; @@ -375,8 +326,6 @@ TEST(RtpPayloadParamsTest, GenericDescriptorForGenericCodec) { } TEST(RtpPayloadParamsTest, SetsGenericFromGenericFrameInfo) { - test::ScopedFieldTrials generic_picture_id( - "WebRTC-GenericDescriptor/Enabled/"); RtpPayloadState state; EncodedImage encoded_image; CodecSpecificInfo codec_info; @@ -388,6 +337,7 @@ TEST(RtpPayloadParamsTest, SetsGenericFromGenericFrameInfo) { GenericFrameInfo::Builder().S(1).T(0).Dtis("S").Build(); codec_info.generic_frame_info->encoder_buffers = { {/*id=*/0, /*referenced=*/false, /*updated=*/true}}; + codec_info.generic_frame_info->part_of_chain = {true, false}; RTPVideoHeader key_header = params.GetRtpVideoHeader(encoded_image, &codec_info, /*frame_id=*/1); @@ -398,12 +348,14 @@ TEST(RtpPayloadParamsTest, SetsGenericFromGenericFrameInfo) { EXPECT_THAT(key_header.generic->dependencies, IsEmpty()); EXPECT_THAT(key_header.generic->decode_target_indications, ElementsAre(DecodeTargetIndication::kSwitch)); + EXPECT_THAT(key_header.generic->chain_diffs, SizeIs(2)); encoded_image._frameType = VideoFrameType::kVideoFrameDelta; codec_info.generic_frame_info = GenericFrameInfo::Builder().S(2).T(3).Dtis("D").Build(); codec_info.generic_frame_info->encoder_buffers = { {/*id=*/0, /*referenced=*/true, /*updated=*/false}}; + codec_info.generic_frame_info->part_of_chain = {false, false}; RTPVideoHeader delta_header = params.GetRtpVideoHeader(encoded_image, &codec_info, /*frame_id=*/3); @@ -414,6 +366,7 @@ TEST(RtpPayloadParamsTest, SetsGenericFromGenericFrameInfo) { EXPECT_THAT(delta_header.generic->dependencies, ElementsAre(1)); EXPECT_THAT(delta_header.generic->decode_target_indications, ElementsAre(DecodeTargetIndication::kDiscardable)); + EXPECT_THAT(delta_header.generic->chain_diffs, SizeIs(2)); } class RtpPayloadParamsVp8ToGenericTest : public ::testing::Test { @@ -421,9 +374,7 @@ class RtpPayloadParamsVp8ToGenericTest : public ::testing::Test { enum LayerSync { kNoSync, kSync }; RtpPayloadParamsVp8ToGenericTest() - : generic_descriptor_field_trial_("WebRTC-GenericDescriptor/Enabled/"), - state_(), - params_(123, &state_, trials_config_) {} + : state_(), params_(123, &state_, trials_config_) {} void ConvertAndCheck(int temporal_index, int64_t shared_frame_id, @@ -459,7 +410,6 @@ class RtpPayloadParamsVp8ToGenericTest : public ::testing::Test { } protected: - test::ScopedFieldTrials generic_descriptor_field_trial_; FieldTrialBasedConfig trials_config_; RtpPayloadState state_; RtpPayloadParams params_; @@ -518,9 +468,7 @@ class RtpPayloadParamsH264ToGenericTest : public ::testing::Test { enum LayerSync { kNoSync, kSync }; RtpPayloadParamsH264ToGenericTest() - : generic_descriptor_field_trial_("WebRTC-GenericDescriptor/Enabled/"), - state_(), - params_(123, &state_, trials_config_) {} + : state_(), params_(123, &state_, trials_config_) {} void ConvertAndCheck(int temporal_index, int64_t shared_frame_id, @@ -556,7 +504,6 @@ class RtpPayloadParamsH264ToGenericTest : public ::testing::Test { } protected: - test::ScopedFieldTrials generic_descriptor_field_trial_; FieldTrialBasedConfig trials_config_; RtpPayloadState state_; RtpPayloadParams params_; diff --git a/chromium/third_party/webrtc/call/rtp_transport_controller_send.cc b/chromium/third_party/webrtc/call/rtp_transport_controller_send.cc index 56c5e55ca17..9baf164a60d 100644 --- a/chromium/third_party/webrtc/call/rtp_transport_controller_send.cc +++ b/chromium/third_party/webrtc/call/rtp_transport_controller_send.cc @@ -91,13 +91,16 @@ RtpTransportControllerSend::RtpTransportControllerSend( event_log, trials, process_thread_.get())), - task_queue_pacer_(use_task_queue_pacer_ - ? new TaskQueuePacedSender(clock, - &packet_router_, - event_log, - trials, - task_queue_factory) - : nullptr), + task_queue_pacer_( + use_task_queue_pacer_ + ? new TaskQueuePacedSender( + clock, + &packet_router_, + event_log, + trials, + task_queue_factory, + /*hold_back_window = */ PacingController::kMinSleepTime) + : nullptr), observer_(nullptr), controller_factory_override_(controller_factory), controller_factory_fallback_( diff --git a/chromium/third_party/webrtc/call/rtp_video_sender.cc b/chromium/third_party/webrtc/call/rtp_video_sender.cc index ca8baee2b09..5f8d2df9655 100644 --- a/chromium/third_party/webrtc/call/rtp_video_sender.cc +++ b/chromium/third_party/webrtc/call/rtp_video_sender.cc @@ -22,8 +22,8 @@ #include "api/video_codecs/video_codec.h" #include "call/rtp_transport_controller_send_interface.h" #include "modules/pacing/packet_router.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "modules/utility/include/process_thread.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -37,7 +37,7 @@ namespace webrtc { namespace webrtc_internal_rtp_video_sender { RtpStreamSender::RtpStreamSender( - std::unique_ptr<RtpRtcp> rtp_rtcp, + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp, std::unique_ptr<RTPSenderVideo> sender_video, std::unique_ptr<VideoFecGenerator> fec_generator) : rtp_rtcp(std::move(rtp_rtcp)), @@ -200,7 +200,7 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders( const WebRtcKeyValueConfig& trials) { RTC_DCHECK_GT(rtp_config.ssrcs.size(), 0); - RtpRtcp::Configuration configuration; + RtpRtcpInterface::Configuration configuration; configuration.clock = clock; configuration.audio = false; configuration.receiver_only = false; @@ -253,7 +253,8 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders( configuration.need_rtp_packet_infos = rtp_config.lntf.enabled; - auto rtp_rtcp = RtpRtcp::Create(configuration); + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp( + ModuleRtpRtcpImpl2::Create(configuration)); rtp_rtcp->SetSendingStatus(false); rtp_rtcp->SetSendingMediaStatus(false); rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound); @@ -282,7 +283,7 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders( video_config.fec_overhead_bytes = fec_generator->MaxPacketOverhead(); } video_config.frame_transformer = frame_transformer; - video_config.worker_queue = transport->GetWorkerQueue()->Get(); + video_config.send_transport_queue = transport->GetWorkerQueue()->Get(); auto sender_video = std::make_unique<RTPSenderVideo>(video_config); rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video), std::move(fec_generator)); @@ -628,7 +629,7 @@ void RtpVideoSender::ConfigureSsrcs() { RTC_CHECK(ssrc_to_rtp_module_.empty()); for (size_t i = 0; i < rtp_config_.ssrcs.size(); ++i) { uint32_t ssrc = rtp_config_.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); + RtpRtcpInterface* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); // Restore RTP state if previous existed. auto it = suspended_ssrcs_.find(ssrc); @@ -645,7 +646,7 @@ void RtpVideoSender::ConfigureSsrcs() { RTC_DCHECK_EQ(rtp_config_.rtx.ssrcs.size(), rtp_config_.ssrcs.size()); for (size_t i = 0; i < rtp_config_.rtx.ssrcs.size(); ++i) { uint32_t ssrc = rtp_config_.rtx.ssrcs[i]; - RtpRtcp* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); + RtpRtcpInterface* const rtp_rtcp = rtp_streams_[i].rtp_rtcp.get(); auto it = suspended_ssrcs_.find(ssrc); if (it != suspended_ssrcs_.end()) rtp_rtcp->SetRtxState(it->second); diff --git a/chromium/third_party/webrtc/call/rtp_video_sender.h b/chromium/third_party/webrtc/call/rtp_video_sender.h index 58bb7f412ef..0c277d6aa71 100644 --- a/chromium/third_party/webrtc/call/rtp_video_sender.h +++ b/chromium/third_party/webrtc/call/rtp_video_sender.h @@ -29,6 +29,7 @@ #include "call/rtp_transport_controller_send_interface.h" #include "call/rtp_video_sender_interface.h" #include "modules/rtp_rtcp/include/flexfec_sender.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/rtp_sender.h" #include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" @@ -44,14 +45,13 @@ namespace webrtc { class FrameEncryptorInterface; class RTPFragmentationHeader; -class RtpRtcp; class RtpTransportControllerSendInterface; namespace webrtc_internal_rtp_video_sender { // RTP state for a single simulcast stream. Internal to the implementation of // RtpVideoSender. struct RtpStreamSender { - RtpStreamSender(std::unique_ptr<RtpRtcp> rtp_rtcp, + RtpStreamSender(std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp, std::unique_ptr<RTPSenderVideo> sender_video, std::unique_ptr<VideoFecGenerator> fec_generator); ~RtpStreamSender(); @@ -60,7 +60,7 @@ struct RtpStreamSender { RtpStreamSender& operator=(RtpStreamSender&&) = default; // Note: Needs pointer stability. - std::unique_ptr<RtpRtcp> rtp_rtcp; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp; std::unique_ptr<RTPSenderVideo> sender_video; std::unique_ptr<VideoFecGenerator> fec_generator; }; @@ -215,7 +215,7 @@ class RtpVideoSender : public RtpVideoSenderInterface, // Effectively const map from SSRC to RtpRtcp, for all media SSRCs. // This map is set at construction time and never changed, but it's // non-trivial to make it properly const. - std::map<uint32_t, RtpRtcp*> ssrc_to_rtp_module_; + std::map<uint32_t, RtpRtcpInterface*> ssrc_to_rtp_module_; RTC_DISALLOW_COPY_AND_ASSIGN(RtpVideoSender); }; diff --git a/chromium/third_party/webrtc/call/rtp_video_sender_unittest.cc b/chromium/third_party/webrtc/call/rtp_video_sender_unittest.cc index a87196111a0..8a88a24e3ba 100644 --- a/chromium/third_party/webrtc/call/rtp_video_sender_unittest.cc +++ b/chromium/third_party/webrtc/call/rtp_video_sender_unittest.cc @@ -56,7 +56,7 @@ const int kDependencyDescriptorExtensionId = 8; class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { public: - MOCK_METHOD1(OnReceivedIntraFrameRequest, void(uint32_t)); + MOCK_METHOD(void, OnReceivedIntraFrameRequest, (uint32_t), (override)); }; RtpSenderObservers CreateObservers( @@ -361,8 +361,10 @@ TEST(RtpVideoSenderTest, CreateWithPreviousStates) { TEST(RtpVideoSenderTest, FrameCountCallbacks) { class MockFrameCountObserver : public FrameCountObserver { public: - MOCK_METHOD2(FrameCountUpdated, - void(const FrameCounts& frame_counts, uint32_t ssrc)); + MOCK_METHOD(void, + FrameCountUpdated, + (const FrameCounts& frame_counts, uint32_t ssrc), + (override)); } callback; RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {}, @@ -676,8 +678,6 @@ TEST(RtpVideoSenderTest, EarlyRetransmits) { } TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { - test::ScopedFieldTrials trials("WebRTC-GenericDescriptor/Enabled/"); - RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.router()->SetActive(true); @@ -705,9 +705,9 @@ TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 1; codec_specific.template_structure->templates = { - GenericFrameInfo::Builder().T(0).Dtis("S").Build(), - GenericFrameInfo::Builder().T(0).Dtis("S").Fdiffs({2}).Build(), - GenericFrameInfo::Builder().T(1).Dtis("D").Fdiffs({1}).Build(), + FrameDependencyTemplate().T(0).Dtis("S"), + FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), + FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), }; // Send two tiny images, mapping to single RTP packets. @@ -742,8 +742,6 @@ TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { } TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { - test::ScopedFieldTrials trials("WebRTC-GenericDescriptor/Enabled/"); - RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); test.router()->SetActive(true); @@ -771,9 +769,9 @@ TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { codec_specific.template_structure.emplace(); codec_specific.template_structure->num_decode_targets = 1; codec_specific.template_structure->templates = { - GenericFrameInfo::Builder().T(0).Dtis("S").Build(), - GenericFrameInfo::Builder().T(0).Dtis("S").Fdiffs({2}).Build(), - GenericFrameInfo::Builder().T(1).Dtis("D").Fdiffs({1}).Build(), + FrameDependencyTemplate().T(0).Dtis("S"), + FrameDependencyTemplate().T(0).Dtis("S").FrameDiffs({2}), + FrameDependencyTemplate().T(1).Dtis("D").FrameDiffs({1}), }; // Send two tiny images, mapping to single RTP packets. diff --git a/chromium/third_party/webrtc/call/test/mock_audio_send_stream.h b/chromium/third_party/webrtc/call/test/mock_audio_send_stream.h index 489e826d0eb..4164dd550e1 100644 --- a/chromium/third_party/webrtc/call/test/mock_audio_send_stream.h +++ b/chromium/third_party/webrtc/call/test/mock_audio_send_stream.h @@ -21,23 +21,26 @@ namespace test { class MockAudioSendStream : public AudioSendStream { public: - MOCK_CONST_METHOD0(GetConfig, const webrtc::AudioSendStream::Config&()); - MOCK_METHOD1(Reconfigure, void(const Config& config)); - MOCK_METHOD0(Start, void()); - MOCK_METHOD0(Stop, void()); + MOCK_METHOD(const webrtc::AudioSendStream::Config&, + GetConfig, + (), + (const, override)); + MOCK_METHOD(void, Reconfigure, (const Config& config), (override)); + MOCK_METHOD(void, Start, (), (override)); + MOCK_METHOD(void, Stop, (), (override)); // GMock doesn't like move-only types, such as std::unique_ptr. - virtual void SendAudioData(std::unique_ptr<webrtc::AudioFrame> audio_frame) { + void SendAudioData(std::unique_ptr<webrtc::AudioFrame> audio_frame) override { SendAudioDataForMock(audio_frame.get()); } - MOCK_METHOD1(SendAudioDataForMock, void(webrtc::AudioFrame* audio_frame)); - MOCK_METHOD4(SendTelephoneEvent, - bool(int payload_type, - int payload_frequency, - int event, - int duration_ms)); - MOCK_METHOD1(SetMuted, void(bool muted)); - MOCK_CONST_METHOD0(GetStats, Stats()); - MOCK_CONST_METHOD1(GetStats, Stats(bool has_remote_tracks)); + MOCK_METHOD(void, SendAudioDataForMock, (webrtc::AudioFrame*)); + MOCK_METHOD( + bool, + SendTelephoneEvent, + (int payload_type, int payload_frequency, int event, int duration_ms), + (override)); + MOCK_METHOD(void, SetMuted, (bool muted), (override)); + MOCK_METHOD(Stats, GetStats, (), (const, override)); + MOCK_METHOD(Stats, GetStats, (bool has_remote_tracks), (const, override)); }; } // namespace test } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/test/mock_bitrate_allocator.h b/chromium/third_party/webrtc/call/test/mock_bitrate_allocator.h index f00ed79c59f..b08916fe4fc 100644 --- a/chromium/third_party/webrtc/call/test/mock_bitrate_allocator.h +++ b/chromium/third_party/webrtc/call/test/mock_bitrate_allocator.h @@ -18,10 +18,15 @@ namespace webrtc { class MockBitrateAllocator : public BitrateAllocatorInterface { public: - MOCK_METHOD2(AddObserver, - void(BitrateAllocatorObserver*, MediaStreamAllocationConfig)); - MOCK_METHOD1(RemoveObserver, void(BitrateAllocatorObserver*)); - MOCK_CONST_METHOD1(GetStartBitrate, int(BitrateAllocatorObserver*)); + MOCK_METHOD(void, + AddObserver, + (BitrateAllocatorObserver*, MediaStreamAllocationConfig), + (override)); + MOCK_METHOD(void, RemoveObserver, (BitrateAllocatorObserver*), (override)); + MOCK_METHOD(int, + GetStartBitrate, + (BitrateAllocatorObserver*), + (const, override)); }; } // namespace webrtc #endif // CALL_TEST_MOCK_BITRATE_ALLOCATOR_H_ diff --git a/chromium/third_party/webrtc/call/test/mock_rtp_packet_sink_interface.h b/chromium/third_party/webrtc/call/test/mock_rtp_packet_sink_interface.h index adc804f941b..e6d14f05c5d 100644 --- a/chromium/third_party/webrtc/call/test/mock_rtp_packet_sink_interface.h +++ b/chromium/third_party/webrtc/call/test/mock_rtp_packet_sink_interface.h @@ -17,7 +17,7 @@ namespace webrtc { class MockRtpPacketSink : public RtpPacketSinkInterface { public: - MOCK_METHOD1(OnRtpPacket, void(const RtpPacketReceived&)); + MOCK_METHOD(void, OnRtpPacket, (const RtpPacketReceived&), (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/call/test/mock_rtp_transport_controller_send.h b/chromium/third_party/webrtc/call/test/mock_rtp_transport_controller_send.h index afc8400f73a..308c087a408 100644 --- a/chromium/third_party/webrtc/call/test/mock_rtp_transport_controller_send.h +++ b/chromium/third_party/webrtc/call/test/mock_rtp_transport_controller_send.h @@ -32,45 +32,73 @@ namespace webrtc { class MockRtpTransportControllerSend : public RtpTransportControllerSendInterface { public: - MOCK_METHOD10( - CreateRtpVideoSender, - RtpVideoSenderInterface*(std::map<uint32_t, RtpState>, - const std::map<uint32_t, RtpPayloadState>&, - const RtpConfig&, - int rtcp_report_interval_ms, - Transport*, - const RtpSenderObservers&, - RtcEventLog*, - std::unique_ptr<FecController>, - const RtpSenderFrameEncryptionConfig&, - rtc::scoped_refptr<FrameTransformerInterface>)); - MOCK_METHOD1(DestroyRtpVideoSender, void(RtpVideoSenderInterface*)); - MOCK_METHOD0(GetWorkerQueue, rtc::TaskQueue*()); - MOCK_METHOD0(packet_router, PacketRouter*()); - MOCK_METHOD0(network_state_estimate_observer, - NetworkStateEstimateObserver*()); - MOCK_METHOD0(transport_feedback_observer, TransportFeedbackObserver*()); - MOCK_METHOD0(packet_sender, RtpPacketSender*()); - MOCK_METHOD1(SetAllocatedSendBitrateLimits, void(BitrateAllocationLimits)); - MOCK_METHOD1(SetPacingFactor, void(float)); - MOCK_METHOD1(SetQueueTimeLimit, void(int)); - MOCK_METHOD0(GetStreamFeedbackProvider, StreamFeedbackProvider*()); - MOCK_METHOD1(RegisterTargetTransferRateObserver, - void(TargetTransferRateObserver*)); - MOCK_METHOD2(OnNetworkRouteChanged, - void(const std::string&, const rtc::NetworkRoute&)); - MOCK_METHOD1(OnNetworkAvailability, void(bool)); - MOCK_METHOD0(GetBandwidthObserver, RtcpBandwidthObserver*()); - MOCK_CONST_METHOD0(GetPacerQueuingDelayMs, int64_t()); - MOCK_CONST_METHOD0(GetFirstPacketTime, absl::optional<Timestamp>()); - MOCK_METHOD1(EnablePeriodicAlrProbing, void(bool)); - MOCK_METHOD1(OnSentPacket, void(const rtc::SentPacket&)); - MOCK_METHOD1(SetSdpBitrateParameters, void(const BitrateConstraints&)); - MOCK_METHOD1(SetClientBitratePreferences, void(const BitrateSettings&)); - MOCK_METHOD1(OnTransportOverheadChanged, void(size_t)); - MOCK_METHOD1(AccountForAudioPacketsInPacedSender, void(bool)); - MOCK_METHOD0(IncludeOverheadInPacedSender, void()); - MOCK_METHOD1(OnReceivedPacket, void(const ReceivedPacket&)); + MOCK_METHOD(RtpVideoSenderInterface*, + CreateRtpVideoSender, + ((std::map<uint32_t, RtpState>), + (const std::map<uint32_t, RtpPayloadState>&), + const RtpConfig&, + int rtcp_report_interval_ms, + Transport*, + const RtpSenderObservers&, + RtcEventLog*, + std::unique_ptr<FecController>, + const RtpSenderFrameEncryptionConfig&, + rtc::scoped_refptr<FrameTransformerInterface>), + (override)); + MOCK_METHOD(void, + DestroyRtpVideoSender, + (RtpVideoSenderInterface*), + (override)); + MOCK_METHOD(rtc::TaskQueue*, GetWorkerQueue, (), (override)); + MOCK_METHOD(PacketRouter*, packet_router, (), (override)); + MOCK_METHOD(NetworkStateEstimateObserver*, + network_state_estimate_observer, + (), + (override)); + MOCK_METHOD(TransportFeedbackObserver*, + transport_feedback_observer, + (), + (override)); + MOCK_METHOD(RtpPacketSender*, packet_sender, (), (override)); + MOCK_METHOD(void, + SetAllocatedSendBitrateLimits, + (BitrateAllocationLimits), + (override)); + MOCK_METHOD(void, SetPacingFactor, (float), (override)); + MOCK_METHOD(void, SetQueueTimeLimit, (int), (override)); + MOCK_METHOD(StreamFeedbackProvider*, + GetStreamFeedbackProvider, + (), + (override)); + MOCK_METHOD(void, + RegisterTargetTransferRateObserver, + (TargetTransferRateObserver*), + (override)); + MOCK_METHOD(void, + OnNetworkRouteChanged, + (const std::string&, const rtc::NetworkRoute&), + (override)); + MOCK_METHOD(void, OnNetworkAvailability, (bool), (override)); + MOCK_METHOD(RtcpBandwidthObserver*, GetBandwidthObserver, (), (override)); + MOCK_METHOD(int64_t, GetPacerQueuingDelayMs, (), (const, override)); + MOCK_METHOD(absl::optional<Timestamp>, + GetFirstPacketTime, + (), + (const, override)); + MOCK_METHOD(void, EnablePeriodicAlrProbing, (bool), (override)); + MOCK_METHOD(void, OnSentPacket, (const rtc::SentPacket&), (override)); + MOCK_METHOD(void, + SetSdpBitrateParameters, + (const BitrateConstraints&), + (override)); + MOCK_METHOD(void, + SetClientBitratePreferences, + (const BitrateSettings&), + (override)); + MOCK_METHOD(void, OnTransportOverheadChanged, (size_t), (override)); + MOCK_METHOD(void, AccountForAudioPacketsInPacedSender, (bool), (override)); + MOCK_METHOD(void, IncludeOverheadInPacedSender, (), (override)); + MOCK_METHOD(void, OnReceivedPacket, (const ReceivedPacket&), (override)); }; } // namespace webrtc #endif // CALL_TEST_MOCK_RTP_TRANSPORT_CONTROLLER_SEND_H_ diff --git a/chromium/third_party/webrtc/call/video_send_stream.h b/chromium/third_party/webrtc/call/video_send_stream.h index 392c955f477..715d5d73e7b 100644 --- a/chromium/third_party/webrtc/call/video_send_stream.h +++ b/chromium/third_party/webrtc/call/video_send_stream.h @@ -18,10 +18,12 @@ #include <vector> #include "absl/types/optional.h" +#include "api/adaptation/resource.h" #include "api/call/transport.h" #include "api/crypto/crypto_options.h" #include "api/frame_transformer_interface.h" #include "api/rtp_parameters.h" +#include "api/scoped_refptr.h" #include "api/video/video_content_type.h" #include "api/video/video_frame.h" #include "api/video/video_sink_interface.h" @@ -215,6 +217,15 @@ class VideoSendStream { // When a stream is stopped, it can't receive, process or deliver packets. virtual void Stop() = 0; + // If the resource is overusing, the VideoSendStream will try to reduce + // resolution or frame rate until no resource is overusing. + // TODO(https://crbug.com/webrtc/11565): When the ResourceAdaptationProcessor + // is moved to Call this method could be deleted altogether in favor of + // Call-level APIs only. + virtual void AddAdaptationResource(rtc::scoped_refptr<Resource> resource) = 0; + virtual std::vector<rtc::scoped_refptr<Resource>> + GetAdaptationResources() = 0; + virtual void SetSource( rtc::VideoSourceInterface<webrtc::VideoFrame>* source, const DegradationPreference& degradation_preference) = 0; |