diff options
Diffstat (limited to 'chromium/third_party/webrtc/modules')
273 files changed, 11792 insertions, 3705 deletions
diff --git a/chromium/third_party/webrtc/modules/BUILD.gn b/chromium/third_party/webrtc/modules/BUILD.gn index ffdd7016a1d..f6f44bd4b36 100644 --- a/chromium/third_party/webrtc/modules/BUILD.gn +++ b/chromium/third_party/webrtc/modules/BUILD.gn @@ -31,10 +31,8 @@ group("modules") { rtc_source_set("module_api_public") { sources = [ "include/module_common_types_public.h" ] - deps = [ - "..:webrtc_common", - "//third_party/abseil-cpp/absl/types:optional", - ] + deps = [ "..:webrtc_common" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("module_api") { diff --git a/chromium/third_party/webrtc/modules/audio_coding/BUILD.gn b/chromium/third_party/webrtc/modules/audio_coding/BUILD.gn index ceee0c0f07e..3480e70df17 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_coding/BUILD.gn @@ -56,6 +56,8 @@ rtc_library("audio_coding") { "../../rtc_base:rtc_base_approved", "../../system_wrappers", "../../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -71,8 +73,8 @@ rtc_library("legacy_encoded_audio_frame") { "../../api/audio_codecs:audio_codecs_api", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("webrtc_cng") { @@ -104,8 +106,8 @@ rtc_library("audio_encoder_cng") { "../../api/units:time_delta", "../../common_audio", "../../rtc_base:checks", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("red") { @@ -122,8 +124,8 @@ rtc_library("red") { "../../common_audio", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("g711") { @@ -143,8 +145,8 @@ rtc_library("g711") { "../../api/units:time_delta", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] public_deps = [ ":g711_c" ] # no-presubmit-check TODO(webrtc:8603) } @@ -175,8 +177,8 @@ rtc_library("g722") { "../../api/units:time_delta", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] public_deps = [ ":g722_c" ] # no-presubmit-check TODO(webrtc:8603) } @@ -208,8 +210,8 @@ rtc_library("ilbc") { "../../common_audio", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] public_deps = [ ":ilbc_c" ] # no-presubmit-check TODO(webrtc:8603) } @@ -384,8 +386,10 @@ rtc_source_set("isac_common") { "../../api/units:time_delta", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", + "../../rtc_base:safe_minmax", + "../../system_wrappers:field_trial", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("isac") { @@ -723,6 +727,8 @@ rtc_library("audio_coding_opus_common") { "../../api/audio_codecs:audio_codecs_api", "../../rtc_base:checks", "../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -751,6 +757,8 @@ rtc_library("webrtc_opus") { "../../rtc_base:rtc_numerics", "../../rtc_base:safe_minmax", "../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -788,6 +796,8 @@ rtc_library("webrtc_multiopus") { "../../rtc_base:rtc_base_approved", "../../rtc_base:safe_minmax", "../../rtc_base:stringutils", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", @@ -848,7 +858,7 @@ rtc_library("audio_network_adaptor_config") { "audio_network_adaptor/audio_network_adaptor_config.cc", "audio_network_adaptor/include/audio_network_adaptor_config.h", ] - deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("audio_network_adaptor") { @@ -874,6 +884,8 @@ rtc_library("audio_network_adaptor") { "audio_network_adaptor/fec_controller_plr_based.h", "audio_network_adaptor/frame_length_controller.cc", "audio_network_adaptor/frame_length_controller.h", + "audio_network_adaptor/frame_length_controller_v2.cc", + "audio_network_adaptor/frame_length_controller_v2.h", "audio_network_adaptor/include/audio_network_adaptor.h", "audio_network_adaptor/util/threshold_curve.h", ] @@ -893,6 +905,9 @@ rtc_library("audio_network_adaptor") { "../../rtc_base/system:file_wrapper", "../../system_wrappers", "../../system_wrappers:field_trial", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/types:optional", ] @@ -995,6 +1010,8 @@ rtc_library("neteq") { "../../system_wrappers", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -1052,8 +1069,8 @@ rtc_library("neteq_tools_minimal") { "../../system_wrappers", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] defines = audio_codec_defines } @@ -1090,8 +1107,8 @@ rtc_library("neteq_test_tools") { "../../test:rtp_test_utils", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] public_deps = [ ":neteq_tools", @@ -1130,6 +1147,8 @@ rtc_library("neteq_tools") { "../../rtc_base:rtc_base_approved", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -1172,8 +1191,8 @@ if (rtc_enable_protobuf) { "../../rtc_base:rtc_base_approved", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] public_deps = # no-presubmit-check TODO(webrtc:8603) [ "../../logging:rtc_event_log_proto" ] } @@ -1230,10 +1249,11 @@ rtc_library("audio_coding_modules_tests_shared") { "../../test:test_support", "../rtp_rtcp:rtp_rtcp_format", "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] - defines = audio_coding_defines if (rtc_enable_protobuf) { @@ -1372,6 +1392,8 @@ if (rtc_include_tests) { "../../system_wrappers", "../../test:fileutils", "../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -1417,9 +1439,9 @@ if (rtc_include_tests) { ":neteq_tools", "../../rtc_base:rtc_base_approved", "../../test:test_support", - "//third_party/abseil-cpp/absl/strings", "//testing/gtest", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("acm_send_test") { @@ -1432,6 +1454,7 @@ if (rtc_include_tests) { defines = audio_coding_defines deps = audio_coding_deps + [ + "//third_party/abseil-cpp/absl/strings", "../../api/audio:audio_frame_api", "../../rtc_base:checks", ":audio_coding", @@ -1497,8 +1520,8 @@ if (rtc_include_tests) { deps = [ "../../rtc_base:checks", "../../test:fileutils", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] sources = [ "neteq/tools/neteq_test_factory.cc", "neteq/tools/neteq_test_factory.h", @@ -1626,15 +1649,14 @@ if (rtc_include_tests) { "../../test:fileutils", "../../test:test_support", "//testing/gtest", - "//third_party/abseil-cpp/absl/flags:flag", ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag" ] } rtc_executable("rtp_encode") { testonly = true deps = audio_coding_deps + [ - "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/flags:parse", ":audio_coding", @@ -1648,6 +1670,7 @@ if (rtc_include_tests) { "../../api/audio_codecs/isac:audio_encoder_isac", "../../api/audio_codecs/opus:audio_encoder_opus", "../../rtc_base:safe_conversions", + "//third_party/abseil-cpp/absl/memory", ] sources = [ "neteq/tools/rtp_encode.cc" ] @@ -1916,6 +1939,7 @@ if (rtc_include_tests) { "audio_network_adaptor/event_log_writer_unittest.cc", "audio_network_adaptor/fec_controller_plr_based_unittest.cc", "audio_network_adaptor/frame_length_controller_unittest.cc", + "audio_network_adaptor/frame_length_controller_v2_unittest.cc", "audio_network_adaptor/util/threshold_curve_unittest.cc", "codecs/builtin_audio_decoder_factory_unittest.cc", "codecs/builtin_audio_encoder_factory_unittest.cc", @@ -2050,8 +2074,11 @@ if (rtc_include_tests) { "codecs/opus/test", "codecs/opus/test:test_unittest", "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/chromium/third_party/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc index 6c9b242e009..b53d456ff7e 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/acm2/audio_coding_module_unittest.cc @@ -252,6 +252,9 @@ class AudioCodingModuleTestOldApi : public ::testing::Test { Clock* clock_; }; +class AudioCodingModuleTestOldApiDeathTest + : public AudioCodingModuleTestOldApi {}; + TEST_F(AudioCodingModuleTestOldApi, VerifyOutputFrame) { AudioFrame audio_frame; const int kSampleRateHz = 32000; @@ -271,7 +274,7 @@ TEST_F(AudioCodingModuleTestOldApi, VerifyOutputFrame) { // http://crbug.com/615050 #if !defined(WEBRTC_WIN) && defined(__clang__) && RTC_DCHECK_IS_ON && \ GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST_F(AudioCodingModuleTestOldApi, FailOnZeroDesiredFrequency) { +TEST_F(AudioCodingModuleTestOldApiDeathTest, FailOnZeroDesiredFrequency) { AudioFrame audio_frame; bool muted; RTC_EXPECT_DEATH(acm_->PlayoutData10Ms(0, &audio_frame, &muted), diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_coding.gni b/chromium/third_party/webrtc/modules/audio_coding/audio_coding.gni index 9b0aba856a4..bf67d9cb8da 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_coding.gni +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_coding.gni @@ -25,9 +25,6 @@ if (current_cpu == "arm") { } else { audio_codec_defines += [ "WEBRTC_CODEC_ISAC" ] } -if (!build_with_mozilla && !build_with_chromium) { - audio_codec_defines += [ "WEBRTC_CODEC_RED" ] -} audio_coding_defines = audio_codec_defines neteq_defines = audio_codec_defines diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/config.proto b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/config.proto index 90c58e5c7d1..347372e8d99 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/config.proto +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/config.proto @@ -1,8 +1,10 @@ syntax = "proto2"; + +package webrtc.audio_network_adaptor.config; + option optimize_for = LITE_RUNTIME; option java_package = "org.webrtc.AudioNetworkAdaptor"; option java_outer_classname = "Config"; -package webrtc.audio_network_adaptor.config; message FecController { message Threshold { @@ -116,6 +118,19 @@ message FrameLengthController { optional int32 fl_60ms_to_40ms_bandwidth_bps = 12; } +message FrameLengthControllerV2 { + // FrameLengthControllerV2 chooses the frame length by taking the target + // bitrate and subtracting the overhead bitrate to obtain the remaining + // bitrate for the payload. The chosen frame length is the shortest possible + // where the payload bitrate is more than |min_payload_bitrate_bps|. + optional int32 min_payload_bitrate_bps = 1; + + // If true, uses the stable target bitrate to decide the frame length. This + // will result in less frame length toggling but spending more time at longer + // frame lengths compared to using the normal target bitrate. + optional bool use_slow_adaptation = 2; +} + message ChannelController { // Uplink bandwidth above which the number of encoded channels should switch // from 1 to 2. @@ -164,6 +179,7 @@ message Controller { DtxController dtx_controller = 24; BitrateController bitrate_controller = 25; FecControllerRplrBased fec_controller_rplr_based = 26; + FrameLengthControllerV2 frame_length_controller_v2 = 27; } } @@ -177,4 +193,3 @@ message ControllerManager { // made. optional float min_reordering_squared_distance = 3; } - diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc index c7aad1da879..415b9fcf52c 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager.cc @@ -11,6 +11,7 @@ #include "modules/audio_coding/audio_network_adaptor/controller_manager.h" #include <cmath> +#include <memory> #include <string> #include <utility> @@ -20,6 +21,7 @@ #include "modules/audio_coding/audio_network_adaptor/dtx_controller.h" #include "modules/audio_coding/audio_network_adaptor/fec_controller_plr_based.h" #include "modules/audio_coding/audio_network_adaptor/frame_length_controller.h" +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" #include "modules/audio_coding/audio_network_adaptor/util/threshold_curve.h" #include "rtc_base/ignore_wundef.h" #include "rtc_base/logging.h" @@ -197,6 +199,14 @@ std::unique_ptr<BitrateController> CreateBitrateController( initial_bitrate_bps, initial_frame_length_ms, fl_increase_overhead_offset, fl_decrease_overhead_offset))); } + +std::unique_ptr<FrameLengthControllerV2> CreateFrameLengthControllerV2( + const audio_network_adaptor::config::FrameLengthControllerV2& config, + rtc::ArrayView<const int> encoder_frame_lengths_ms) { + return std::make_unique<FrameLengthControllerV2>( + encoder_frame_lengths_ms, config.min_payload_bitrate_bps(), + config.use_slow_adaptation()); +} #endif // WEBRTC_ENABLE_PROTOBUF } // namespace @@ -277,6 +287,11 @@ std::unique_ptr<ControllerManager> ControllerManagerImpl::Create( controller_config.bitrate_controller(), initial_bitrate_bps, initial_frame_length_ms); break; + case audio_network_adaptor::config::Controller::kFrameLengthControllerV2: + controller = CreateFrameLengthControllerV2( + controller_config.frame_length_controller_v2(), + encoder_frame_lengths_ms); + break; default: RTC_NOTREACHED(); } diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc index 4286434b5b9..c71bbc9e2ad 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/controller_manager_unittest.cc @@ -260,6 +260,14 @@ void AddFrameLengthControllerConfig( kChracteristicPacketLossFraction[1]); } +void AddFrameLengthControllerV2Config( + audio_network_adaptor::config::ControllerManager* config) { + auto controller = + config->add_controllers()->mutable_frame_length_controller_v2(); + controller->set_min_payload_bitrate_bps(16000); + controller->set_use_slow_adaptation(true); +} + constexpr int kInitialBitrateBps = 24000; constexpr size_t kIntialChannelsToEncode = 1; constexpr bool kInitialDtxEnabled = true; @@ -464,6 +472,14 @@ TEST(ControllerManagerTest, CreateFromConfigStringAndCheckReordering) { ControllerType::CHANNEL, ControllerType::DTX, ControllerType::BIT_RATE}); } + +TEST(ControllerManagerTest, CreateFrameLengthControllerV2) { + audio_network_adaptor::config::ControllerManager config; + AddFrameLengthControllerV2Config(&config); + auto states = CreateControllerManager(config.SerializeAsString()); + auto controllers = states.controller_manager->GetControllers(); + EXPECT_TRUE(controllers.size() == 1); +} #endif // WEBRTC_ENABLE_PROTOBUF } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc new file mode 100644 index 00000000000..36fc10ba825 --- /dev/null +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.cc @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" + +#include <algorithm> + +#include "absl/algorithm/container.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +int OverheadBps(int overhead_bytes_per_packet, int frame_length_ms) { + return overhead_bytes_per_packet * 8 * 1000 / frame_length_ms; +} + +} // namespace + +FrameLengthControllerV2::FrameLengthControllerV2( + rtc::ArrayView<const int> encoder_frame_lengths_ms, + int min_payload_bitrate_bps, + bool use_slow_adaptation) + : encoder_frame_lengths_ms_(encoder_frame_lengths_ms.begin(), + encoder_frame_lengths_ms.end()), + min_payload_bitrate_bps_(min_payload_bitrate_bps), + use_slow_adaptation_(use_slow_adaptation) { + RTC_CHECK(!encoder_frame_lengths_ms_.empty()); + absl::c_sort(encoder_frame_lengths_ms_); +} + +void FrameLengthControllerV2::UpdateNetworkMetrics( + const NetworkMetrics& network_metrics) { + if (network_metrics.target_audio_bitrate_bps) { + target_bitrate_bps_ = network_metrics.target_audio_bitrate_bps; + } + if (network_metrics.overhead_bytes_per_packet) { + overhead_bytes_per_packet_ = network_metrics.overhead_bytes_per_packet; + } + if (network_metrics.uplink_bandwidth_bps) { + uplink_bandwidth_bps_ = network_metrics.uplink_bandwidth_bps; + } +} + +void FrameLengthControllerV2::MakeDecision(AudioEncoderRuntimeConfig* config) { + if (!target_bitrate_bps_ || !overhead_bytes_per_packet_ || + !uplink_bandwidth_bps_) { + return; + } + + auto it = + absl::c_find_if(encoder_frame_lengths_ms_, [&](int frame_length_ms) { + int target = use_slow_adaptation_ ? *uplink_bandwidth_bps_ + : *target_bitrate_bps_; + return target - + OverheadBps(*overhead_bytes_per_packet_, frame_length_ms) > + min_payload_bitrate_bps_; + }); + + // Longest frame length is chosen if none match our criteria. + config->frame_length_ms = it != encoder_frame_lengths_ms_.end() + ? *it + : encoder_frame_lengths_ms_.back(); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h new file mode 100644 index 00000000000..d7102b0b44d --- /dev/null +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ +#define MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ + +#include <vector> + +#include "absl/types/optional.h" +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor.h" + +namespace webrtc { + +class FrameLengthControllerV2 final : public Controller { + public: + FrameLengthControllerV2(rtc::ArrayView<const int> encoder_frame_lengths_ms, + int min_payload_bitrate_bps, + bool use_slow_adaptation); + + void UpdateNetworkMetrics(const NetworkMetrics& network_metrics) override; + + void MakeDecision(AudioEncoderRuntimeConfig* config) override; + + private: + std::vector<int> encoder_frame_lengths_ms_; + const int min_payload_bitrate_bps_; + const bool use_slow_adaptation_; + + absl::optional<int> uplink_bandwidth_bps_; + absl::optional<int> target_bitrate_bps_; + absl::optional<int> overhead_bytes_per_packet_; +}; + +} // namespace webrtc + +#endif // MODULES_AUDIO_CODING_AUDIO_NETWORK_ADAPTOR_FRAME_LENGTH_CONTROLLER_V2_H_ diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc new file mode 100644 index 00000000000..1c88f47c583 --- /dev/null +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/frame_length_controller_v2_unittest.cc @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/audio_coding/audio_network_adaptor/frame_length_controller_v2.h" + +#include <algorithm> +#include <memory> + +#include "modules/audio_coding/audio_network_adaptor/controller.h" +#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +constexpr int kANASupportedFrameLengths[] = {20, 40, 60, 120}; +constexpr int kMinPayloadBitrateBps = 16000; + +} // namespace + +class FrameLengthControllerV2Test : public testing::Test { + protected: + AudioEncoderRuntimeConfig GetDecision() { + AudioEncoderRuntimeConfig config; + controller_->MakeDecision(&config); + return config; + } + + void SetOverhead(int overhead_bytes_per_packet) { + overhead_bytes_per_packet_ = overhead_bytes_per_packet; + Controller::NetworkMetrics metrics; + metrics.overhead_bytes_per_packet = overhead_bytes_per_packet; + controller_->UpdateNetworkMetrics(metrics); + } + + void SetTargetBitrate(int target_audio_bitrate_bps) { + target_audio_bitrate_bps_ = target_audio_bitrate_bps; + Controller::NetworkMetrics metrics; + metrics.target_audio_bitrate_bps = target_audio_bitrate_bps; + controller_->UpdateNetworkMetrics(metrics); + } + + void SetUplinkBandwidth(int uplink_bandwidth_bps) { + Controller::NetworkMetrics metrics; + metrics.uplink_bandwidth_bps = uplink_bandwidth_bps; + controller_->UpdateNetworkMetrics(metrics); + } + + void ExpectFrameLengthDecision(int expected_frame_length_ms) { + auto config = GetDecision(); + EXPECT_EQ(*config.frame_length_ms, expected_frame_length_ms); + } + + std::unique_ptr<FrameLengthControllerV2> controller_ = + std::make_unique<FrameLengthControllerV2>(kANASupportedFrameLengths, + kMinPayloadBitrateBps, + /*use_slow_adaptation=*/false); + absl::optional<int> target_audio_bitrate_bps_; + absl::optional<int> overhead_bytes_per_packet_; +}; + +// Don't return any decision if we haven't received all required network +// metrics. +TEST_F(FrameLengthControllerV2Test, RequireNetworkMetrics) { + auto config = GetDecision(); + EXPECT_FALSE(config.bitrate_bps); + EXPECT_FALSE(config.frame_length_ms); + + SetOverhead(30); + config = GetDecision(); + EXPECT_FALSE(config.frame_length_ms); + + SetTargetBitrate(32000); + config = GetDecision(); + EXPECT_FALSE(config.frame_length_ms); + + SetUplinkBandwidth(32000); + config = GetDecision(); + EXPECT_TRUE(config.frame_length_ms); +} + +TEST_F(FrameLengthControllerV2Test, UseFastAdaptation) { + SetOverhead(50); + SetTargetBitrate(50000); + SetUplinkBandwidth(50000); + ExpectFrameLengthDecision(20); + + SetTargetBitrate(20000); + ExpectFrameLengthDecision(120); + + SetTargetBitrate(30000); + ExpectFrameLengthDecision(40); + + SetTargetBitrate(25000); + ExpectFrameLengthDecision(60); +} + +TEST_F(FrameLengthControllerV2Test, UseSlowAdaptation) { + controller_ = std::make_unique<FrameLengthControllerV2>( + kANASupportedFrameLengths, kMinPayloadBitrateBps, + /*use_slow_adaptation=*/true); + SetOverhead(50); + SetTargetBitrate(50000); + SetUplinkBandwidth(20000); + ExpectFrameLengthDecision(120); + + SetUplinkBandwidth(30000); + ExpectFrameLengthDecision(40); + + SetUplinkBandwidth(40000); + ExpectFrameLengthDecision(20); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h index 8c048496cad..26a9061745d 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_audio_network_adaptor.h @@ -18,27 +18,38 @@ namespace webrtc { class MockAudioNetworkAdaptor : public AudioNetworkAdaptor { public: - virtual ~MockAudioNetworkAdaptor() { Die(); } - MOCK_METHOD0(Die, void()); + ~MockAudioNetworkAdaptor() override { Die(); } + MOCK_METHOD(void, Die, ()); - MOCK_METHOD1(SetUplinkBandwidth, void(int uplink_bandwidth_bps)); + MOCK_METHOD(void, SetUplinkBandwidth, (int uplink_bandwidth_bps), (override)); - MOCK_METHOD1(SetUplinkPacketLossFraction, - void(float uplink_packet_loss_fraction)); + MOCK_METHOD(void, + SetUplinkPacketLossFraction, + (float uplink_packet_loss_fraction), + (override)); - MOCK_METHOD1(SetRtt, void(int rtt_ms)); + MOCK_METHOD(void, SetRtt, (int rtt_ms), (override)); - MOCK_METHOD1(SetTargetAudioBitrate, void(int target_audio_bitrate_bps)); + MOCK_METHOD(void, + SetTargetAudioBitrate, + (int target_audio_bitrate_bps), + (override)); - MOCK_METHOD1(SetOverhead, void(size_t overhead_bytes_per_packet)); + MOCK_METHOD(void, + SetOverhead, + (size_t overhead_bytes_per_packet), + (override)); - MOCK_METHOD0(GetEncoderRuntimeConfig, AudioEncoderRuntimeConfig()); + MOCK_METHOD(AudioEncoderRuntimeConfig, + GetEncoderRuntimeConfig, + (), + (override)); - MOCK_METHOD1(StartDebugDump, void(FILE* file_handle)); + MOCK_METHOD(void, StartDebugDump, (FILE * file_handle), (override)); - MOCK_METHOD0(StopDebugDump, void()); + MOCK_METHOD(void, StopDebugDump, (), (override)); - MOCK_CONST_METHOD0(GetStats, ANAStats()); + MOCK_METHOD(ANAStats, GetStats, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h index df28e9e26f1..de554c05177 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller.h @@ -18,11 +18,16 @@ namespace webrtc { class MockController : public Controller { public: - virtual ~MockController() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD1(UpdateNetworkMetrics, - void(const NetworkMetrics& network_metrics)); - MOCK_METHOD1(MakeDecision, void(AudioEncoderRuntimeConfig* config)); + ~MockController() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, + UpdateNetworkMetrics, + (const NetworkMetrics& network_metrics), + (override)); + MOCK_METHOD(void, + MakeDecision, + (AudioEncoderRuntimeConfig * config), + (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h index 8d410a742d6..9e2fa466fc0 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_controller_manager.h @@ -20,12 +20,13 @@ namespace webrtc { class MockControllerManager : public ControllerManager { public: - virtual ~MockControllerManager() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD1( - GetSortedControllers, - std::vector<Controller*>(const Controller::NetworkMetrics& metrics)); - MOCK_CONST_METHOD0(GetControllers, std::vector<Controller*>()); + ~MockControllerManager() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(std::vector<Controller*>, + GetSortedControllers, + (const Controller::NetworkMetrics& metrics), + (override)); + MOCK_METHOD(std::vector<Controller*>, GetControllers, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h index 06650abbd60..0c6a9efe1d2 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/mock/mock_debug_dump_writer.h @@ -18,20 +18,24 @@ namespace webrtc { class MockDebugDumpWriter : public DebugDumpWriter { public: - virtual ~MockDebugDumpWriter() { Die(); } - MOCK_METHOD0(Die, void()); + ~MockDebugDumpWriter() override { Die(); } + MOCK_METHOD(void, Die, ()); - MOCK_METHOD2(DumpEncoderRuntimeConfig, - void(const AudioEncoderRuntimeConfig& config, - int64_t timestamp)); - MOCK_METHOD2(DumpNetworkMetrics, - void(const Controller::NetworkMetrics& metrics, - int64_t timestamp)); + MOCK_METHOD(void, + DumpEncoderRuntimeConfig, + (const AudioEncoderRuntimeConfig& config, int64_t timestamp), + (override)); + MOCK_METHOD(void, + DumpNetworkMetrics, + (const Controller::NetworkMetrics& metrics, int64_t timestamp), + (override)); #if WEBRTC_ENABLE_PROTOBUF - MOCK_METHOD2(DumpControllerManagerConfig, - void(const audio_network_adaptor::config::ControllerManager& - controller_manager_config, - int64_t timestamp)); + MOCK_METHOD(void, + DumpControllerManagerConfig, + (const audio_network_adaptor::config::ControllerManager& + controller_manager_config, + int64_t timestamp), + (override)); #endif }; diff --git a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc index 9984049d501..dc3aec0b185 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/audio_network_adaptor/util/threshold_curve_unittest.cc @@ -621,7 +621,7 @@ TEST(ThresholdCurveTest, NearlyIdenticalCurvesSecondContinuesOnOtherRightSide) { // The higher-left point must be given as the first point, and the lower-right // point must be given as the second. // This necessarily produces a non-positive slope. -TEST(ThresholdCurveTest, WrongOrderPoints) { +TEST(ThresholdCurveDeathTest, WrongOrderPoints) { std::unique_ptr<ThresholdCurve> curve; constexpr ThresholdCurve::Point left{5, 10}; constexpr ThresholdCurve::Point right{10, 5}; diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc index 80349e25047..0e6ab793943 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/cng/cng_unittest.cc @@ -40,6 +40,8 @@ class CngTest : public ::testing::Test { int16_t speech_data_[640]; // Max size of CNG internal buffers. }; +class CngDeathTest : public CngTest {}; + void CngTest::SetUp() { FILE* input_file; const std::string file_name = @@ -69,7 +71,7 @@ void CngTest::TestCngEncode(int sample_rate_hz, int quality) { #if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Create CNG encoder, init with faulty values, free CNG encoder. -TEST_F(CngTest, CngInitFail) { +TEST_F(CngDeathTest, CngInitFail) { // Call with too few parameters. EXPECT_DEATH( { @@ -86,7 +88,7 @@ TEST_F(CngTest, CngInitFail) { } // Encode Cng with too long input vector. -TEST_F(CngTest, CngEncodeTooLong) { +TEST_F(CngDeathTest, CngEncodeTooLong) { rtc::Buffer sid_data; // Create encoder. diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h b/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h index a3b8e76a30e..d99e9c893f8 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t.h @@ -19,6 +19,7 @@ #include "api/scoped_refptr.h" #include "api/units/time_delta.h" #include "rtc_base/constructor_magic.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { @@ -48,6 +49,13 @@ class AudioEncoderIsacT final : public AudioEncoder { size_t Num10MsFramesInNextPacket() const override; size_t Max10MsFramesInAPacket() const override; int GetTargetBitrate() const override; + void SetTargetBitrate(int target_bps) override; + void OnReceivedTargetAudioBitrate(int target_bps) override; + void OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional<int64_t> bwe_period_ms) override; + void OnReceivedUplinkAllocation(BitrateAllocationUpdate update) override; + void OnReceivedOverhead(size_t overhead_bytes_per_packet) override; EncodedInfo EncodeImpl(uint32_t rtp_timestamp, rtc::ArrayView<const int16_t> audio, rtc::Buffer* encoded) override; @@ -60,7 +68,13 @@ class AudioEncoderIsacT final : public AudioEncoder { // STREAM_MAXW16_60MS for iSAC fix (60 ms). static const size_t kSufficientEncodeBufferSizeBytes = 400; - static const int kDefaultBitRate = 32000; + static constexpr int kDefaultBitRate = 32000; + static constexpr int kMinBitrateBps = 10000; + static constexpr int MaxBitrateBps(int sample_rate_hz) { + return sample_rate_hz == 32000 ? 56000 : 32000; + } + + void SetTargetBitrate(int target_bps, bool subtract_per_packet_overhead); // Recreate the iSAC encoder instance with the given settings, and save them. void RecreateEncoderInstance(const Config& config); @@ -77,6 +91,15 @@ class AudioEncoderIsacT final : public AudioEncoder { // Timestamp of the previously encoded packet. uint32_t last_encoded_timestamp_; + // Cache the value of the "WebRTC-SendSideBwe-WithOverhead" field trial. + const bool send_side_bwe_with_overhead_ = + field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead"); + + // When we send a packet, expect this many bytes of headers to be added to it. + // Start out with a reasonable default that we can use until we receive a real + // value. + DataSize overhead_per_packet_ = DataSize::Bytes(28); + RTC_DISALLOW_COPY_AND_ASSIGN(AudioEncoderIsacT); }; diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h b/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h index 9ddb94326d1..0bde3f797f2 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/audio_encoder_isac_t_impl.h @@ -12,6 +12,7 @@ #define MODULES_AUDIO_CODING_CODECS_ISAC_AUDIO_ENCODER_ISAC_T_IMPL_H_ #include "rtc_base/checks.h" +#include "rtc_base/numerics/safe_minmax.h" namespace webrtc { @@ -81,6 +82,51 @@ int AudioEncoderIsacT<T>::GetTargetBitrate() const { } template <typename T> +void AudioEncoderIsacT<T>::SetTargetBitrate(int target_bps) { + // Set target bitrate directly without subtracting per-packet overhead, + // because that's what AudioEncoderOpus does. + SetTargetBitrate(target_bps, + /*subtract_per_packet_overhead=*/false); +} + +template <typename T> +void AudioEncoderIsacT<T>::OnReceivedTargetAudioBitrate(int target_bps) { + // Set target bitrate directly without subtracting per-packet overhead, + // because that's what AudioEncoderOpus does. + SetTargetBitrate(target_bps, + /*subtract_per_packet_overhead=*/false); +} + +template <typename T> +void AudioEncoderIsacT<T>::OnReceivedUplinkBandwidth( + int target_audio_bitrate_bps, + absl::optional<int64_t> /*bwe_period_ms*/) { + // Set target bitrate, subtracting the per-packet overhead if + // WebRTC-SendSideBwe-WithOverhead is enabled, because that's what + // AudioEncoderOpus does. + SetTargetBitrate( + target_audio_bitrate_bps, + /*subtract_per_packet_overhead=*/send_side_bwe_with_overhead_); +} + +template <typename T> +void AudioEncoderIsacT<T>::OnReceivedUplinkAllocation( + BitrateAllocationUpdate update) { + // Set target bitrate, subtracting the per-packet overhead if + // WebRTC-SendSideBwe-WithOverhead is enabled, because that's what + // AudioEncoderOpus does. + SetTargetBitrate( + update.target_bitrate.bps<int>(), + /*subtract_per_packet_overhead=*/send_side_bwe_with_overhead_); +} + +template <typename T> +void AudioEncoderIsacT<T>::OnReceivedOverhead( + size_t overhead_bytes_per_packet) { + overhead_per_packet_ = DataSize::Bytes(overhead_bytes_per_packet); +} + +template <typename T> AudioEncoder::EncodedInfo AudioEncoderIsacT<T>::EncodeImpl( uint32_t rtp_timestamp, rtc::ArrayView<const int16_t> audio, @@ -127,6 +173,21 @@ AudioEncoderIsacT<T>::GetFrameLengthRange() const { } template <typename T> +void AudioEncoderIsacT<T>::SetTargetBitrate(int target_bps, + bool subtract_per_packet_overhead) { + if (subtract_per_packet_overhead) { + const DataRate overhead_rate = + overhead_per_packet_ / TimeDelta::Millis(config_.frame_size_ms); + target_bps -= overhead_rate.bps(); + } + target_bps = rtc::SafeClamp(target_bps, kMinBitrateBps, + MaxBitrateBps(config_.sample_rate_hz)); + int result = T::Control(isac_state_, target_bps, config_.frame_size_ms); + RTC_DCHECK_EQ(result, 0); + config_.bit_rate = target_bps; +} + +template <typename T> void AudioEncoderIsacT<T>::RecreateEncoderInstance(const Config& config) { RTC_CHECK(config.IsOk()); packet_in_progress_ = false; diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/isac_webrtc_api_test.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/isac_webrtc_api_test.cc index c4d7ab8fa80..a2e1e088e6e 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/isac_webrtc_api_test.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/isac/isac_webrtc_api_test.cc @@ -9,6 +9,7 @@ */ #include <array> +#include <map> #include <memory> #include <vector> @@ -159,6 +160,33 @@ TEST_P(EncoderTest, TestDifferentBitrates) { EXPECT_LT(num_bytes_low, num_bytes_high); } +// Encodes an input audio sequence first with a low, then with a high target +// bitrate *using the same encoder* and checks that the number of emitted bytes +// in the first case is less than in the second case. +TEST_P(EncoderTest, TestDynamicBitrateChange) { + constexpr int kLowBps = 20000; + constexpr int kHighBps = 25000; + constexpr int kStartBps = 30000; + auto encoder = CreateEncoder(GetIsacImpl(), GetSampleRateHz(), + GetFrameSizeMs(), kStartBps); + std::map<int, int> num_bytes; + constexpr int kNumFrames = 200; // 2 seconds. + for (int bitrate_bps : {kLowBps, kHighBps}) { + auto pcm_file = GetPcmTestFileReader(GetSampleRateHz()); + encoder->OnReceivedTargetAudioBitrate(bitrate_bps); + for (int i = 0; i < kNumFrames; ++i) { + AudioFrame in; + pcm_file->Read10MsData(in); + rtc::Buffer buf; + encoder->Encode(/*rtp_timestamp=*/0, AudioFrameToView(in), &buf); + num_bytes[bitrate_bps] += buf.size(); + } + } + // kHighBps / kLowBps == 1.25, so require the high-bitrate run to produce at + // least 1.2 times the number of bytes. + EXPECT_LT(1.2 * num_bytes[kLowBps], num_bytes[kHighBps]); +} + // Checks that, given a target bitrate, the encoder does not overshoot too much. TEST_P(EncoderTest, DoNotOvershootTargetBitrate) { for (int bitrate_bps : {10000, 15000, 20000, 26000, 32000}) { diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc index ef32f4ce02d..220e96f1b77 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.cc @@ -66,46 +66,7 @@ constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60}; // PacketLossFractionSmoother uses an exponential filter with a time constant // of -1.0 / ln(0.9999) = 10000 ms. constexpr float kAlphaForPacketLossFractionSmoother = 0.9999f; - -// Optimize the loss rate to configure Opus. Basically, optimized loss rate is -// the input loss rate rounded down to various levels, because a robustly good -// audio quality is achieved by lowering the packet loss down. -// Additionally, to prevent toggling, margins are used, i.e., when jumping to -// a loss rate from below, a higher threshold is used than jumping to the same -// level from above. -float OptimizePacketLossRate(float new_loss_rate, float old_loss_rate) { - RTC_DCHECK_GE(new_loss_rate, 0.0f); - RTC_DCHECK_LE(new_loss_rate, 1.0f); - RTC_DCHECK_GE(old_loss_rate, 0.0f); - RTC_DCHECK_LE(old_loss_rate, 1.0f); - constexpr float kPacketLossRate20 = 0.20f; - constexpr float kPacketLossRate10 = 0.10f; - constexpr float kPacketLossRate5 = 0.05f; - constexpr float kPacketLossRate1 = 0.01f; - constexpr float kLossRate20Margin = 0.02f; - constexpr float kLossRate10Margin = 0.01f; - constexpr float kLossRate5Margin = 0.01f; - if (new_loss_rate >= - kPacketLossRate20 + - kLossRate20Margin * - (kPacketLossRate20 - old_loss_rate > 0 ? 1 : -1)) { - return kPacketLossRate20; - } else if (new_loss_rate >= - kPacketLossRate10 + - kLossRate10Margin * - (kPacketLossRate10 - old_loss_rate > 0 ? 1 : -1)) { - return kPacketLossRate10; - } else if (new_loss_rate >= - kPacketLossRate5 + - kLossRate5Margin * - (kPacketLossRate5 - old_loss_rate > 0 ? 1 : -1)) { - return kPacketLossRate5; - } else if (new_loss_rate >= kPacketLossRate1) { - return kPacketLossRate1; - } else { - return 0.0f; - } -} +constexpr float kMaxPacketLossFraction = 0.2f; int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) { const int bitrate = [&] { @@ -201,35 +162,6 @@ int GetBitrateBps(const AudioEncoderOpusConfig& config) { return *config.bitrate_bps; } -bool IsValidPacketLossRate(int value) { - return value >= 0 && value <= 100; -} - -float ToFraction(int percent) { - return static_cast<float>(percent) / 100; -} - -float GetMinPacketLossRate() { - constexpr char kPacketLossFieldTrial[] = "WebRTC-Audio-OpusMinPacketLossRate"; - const bool use_opus_min_packet_loss_rate = - webrtc::field_trial::IsEnabled(kPacketLossFieldTrial); - if (use_opus_min_packet_loss_rate) { - const std::string field_trial_string = - webrtc::field_trial::FindFullName(kPacketLossFieldTrial); - constexpr int kDefaultMinPacketLossRate = 1; - int value = kDefaultMinPacketLossRate; - if (sscanf(field_trial_string.c_str(), "Enabled-%d", &value) == 1 && - !IsValidPacketLossRate(value)) { - RTC_LOG(LS_WARNING) << "Invalid parameter for " << kPacketLossFieldTrial - << ", using default value: " - << kDefaultMinPacketLossRate; - value = kDefaultMinPacketLossRate; - } - return ToFraction(value); - } - return 0.0; -} - std::vector<float> GetBitrateMultipliers() { constexpr char kBitrateMultipliersName[] = "WebRTC-Audio-OpusBitrateMultipliers"; @@ -425,14 +357,13 @@ AudioEncoderOpusImpl::AudioEncoderOpusImpl( : payload_type_(payload_type), send_side_bwe_with_overhead_( webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")), - use_stable_target_for_adaptation_(webrtc::field_trial::IsEnabled( + use_stable_target_for_adaptation_(!webrtc::field_trial::IsDisabled( "WebRTC-Audio-StableTargetAdaptation")), adjust_bandwidth_( webrtc::field_trial::IsEnabled("WebRTC-AdjustOpusBandwidth")), bitrate_changed_(true), bitrate_multipliers_(GetBitrateMultipliers()), packet_loss_rate_(0.0), - min_packet_loss_rate_(GetMinPacketLossRate()), inst_(nullptr), packet_loss_fraction_smoother_(new PacketLossFractionSmoother()), audio_network_adaptor_creator_(audio_network_adaptor_creator), @@ -541,14 +472,14 @@ void AudioEncoderOpusImpl::DisableAudioNetworkAdaptor() { void AudioEncoderOpusImpl::OnReceivedUplinkPacketLossFraction( float uplink_packet_loss_fraction) { - if (!audio_network_adaptor_) { - packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction); - float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage(); - return SetProjectedPacketLossRate(average_fraction_loss); + if (audio_network_adaptor_) { + audio_network_adaptor_->SetUplinkPacketLossFraction( + uplink_packet_loss_fraction); + ApplyAudioNetworkAdaptor(); } - audio_network_adaptor_->SetUplinkPacketLossFraction( - uplink_packet_loss_fraction); - ApplyAudioNetworkAdaptor(); + packet_loss_fraction_smoother_->AddSample(uplink_packet_loss_fraction); + float average_fraction_loss = packet_loss_fraction_smoother_->GetAverage(); + SetProjectedPacketLossRate(average_fraction_loss); } void AudioEncoderOpusImpl::OnReceivedTargetAudioBitrate( @@ -789,8 +720,7 @@ void AudioEncoderOpusImpl::SetNumChannelsToEncode( } void AudioEncoderOpusImpl::SetProjectedPacketLossRate(float fraction) { - fraction = OptimizePacketLossRate(fraction, packet_loss_rate_); - fraction = std::max(fraction, min_packet_loss_rate_); + fraction = std::min(std::max(fraction, 0.0f), kMaxPacketLossFraction); if (packet_loss_rate_ != fraction) { packet_loss_rate_ = fraction; RTC_CHECK_EQ( @@ -828,10 +758,6 @@ void AudioEncoderOpusImpl::ApplyAudioNetworkAdaptor() { SetTargetBitrate(*config.bitrate_bps); if (config.frame_length_ms) SetFrameLength(*config.frame_length_ms); - if (config.enable_fec) - SetFec(*config.enable_fec); - if (config.uplink_packet_loss_fraction) - SetProjectedPacketLossRate(*config.uplink_packet_loss_fraction); if (config.enable_dtx) SetDtx(*config.enable_dtx); if (config.num_channels) diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h index 540413290d7..ab954feba78 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus.h @@ -160,7 +160,6 @@ class AudioEncoderOpusImpl final : public AudioEncoder { // 1 kbps range. std::vector<float> bitrate_multipliers_; float packet_loss_rate_; - const float min_packet_loss_rate_; std::vector<int16_t> input_buffer_; OpusEncInst* inst_; uint32_t first_timestamp_in_buffer_; diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc index b469885c1f0..1cbc4a3ff70 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/audio_encoder_opus_unittest.cc @@ -93,17 +93,13 @@ std::unique_ptr<AudioEncoderOpusStates> CreateCodec(int sample_rate_hz, AudioEncoderRuntimeConfig CreateEncoderRuntimeConfig() { constexpr int kBitrate = 40000; constexpr int kFrameLength = 60; - constexpr bool kEnableFec = true; constexpr bool kEnableDtx = false; constexpr size_t kNumChannels = 1; - constexpr float kPacketLossFraction = 0.1f; AudioEncoderRuntimeConfig config; config.bitrate_bps = kBitrate; config.frame_length_ms = kFrameLength; - config.enable_fec = kEnableFec; config.enable_dtx = kEnableDtx; config.num_channels = kNumChannels; - config.uplink_packet_loss_fraction = kPacketLossFraction; return config; } @@ -111,7 +107,6 @@ void CheckEncoderRuntimeConfig(const AudioEncoderOpusImpl* encoder, const AudioEncoderRuntimeConfig& config) { EXPECT_EQ(*config.bitrate_bps, encoder->GetTargetBitrate()); EXPECT_EQ(*config.frame_length_ms, encoder->next_frame_length_ms()); - EXPECT_EQ(*config.enable_fec, encoder->fec_enabled()); EXPECT_EQ(*config.enable_dtx, encoder->GetDtx()); EXPECT_EQ(*config.num_channels, encoder->num_channels_to_encode()); } @@ -222,84 +217,6 @@ TEST_P(AudioEncoderOpusTest, } } -namespace { - -// Returns a vector with the n evenly-spaced numbers a, a + (b - a)/(n - 1), -// ..., b. -std::vector<float> IntervalSteps(float a, float b, size_t n) { - RTC_DCHECK_GT(n, 1u); - const float step = (b - a) / (n - 1); - std::vector<float> points; - points.push_back(a); - for (size_t i = 1; i < n - 1; ++i) - points.push_back(a + i * step); - points.push_back(b); - return points; -} - -// Sets the packet loss rate to each number in the vector in turn, and verifies -// that the loss rate as reported by the encoder is |expected_return| for all -// of them. -void TestSetPacketLossRate(const AudioEncoderOpusStates* states, - const std::vector<float>& losses, - float expected_return) { - // |kSampleIntervalMs| is chosen to ease the calculation since - // 0.9999 ^ 184198 = 1e-8. Which minimizes the effect of - // PacketLossFractionSmoother used in AudioEncoderOpus. - constexpr int64_t kSampleIntervalMs = 184198; - for (float loss : losses) { - states->encoder->OnReceivedUplinkPacketLossFraction(loss); - states->fake_clock->AdvanceTime(TimeDelta::Millis(kSampleIntervalMs)); - EXPECT_FLOAT_EQ(expected_return, states->encoder->packet_loss_rate()); - } -} - -} // namespace - -TEST_P(AudioEncoderOpusTest, PacketLossRateOptimized) { - auto states = CreateCodec(sample_rate_hz_, 1); - auto I = [](float a, float b) { return IntervalSteps(a, b, 10); }; - constexpr float eps = 1e-8f; - - // Note that the order of the following calls is critical. - - // clang-format off - TestSetPacketLossRate(states.get(), I(0.00f , 0.01f - eps), 0.00f); - TestSetPacketLossRate(states.get(), I(0.01f + eps, 0.06f - eps), 0.01f); - TestSetPacketLossRate(states.get(), I(0.06f + eps, 0.11f - eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.11f + eps, 0.22f - eps), 0.10f); - TestSetPacketLossRate(states.get(), I(0.22f + eps, 1.00f ), 0.20f); - - TestSetPacketLossRate(states.get(), I(1.00f , 0.18f + eps), 0.20f); - TestSetPacketLossRate(states.get(), I(0.18f - eps, 0.09f + eps), 0.10f); - TestSetPacketLossRate(states.get(), I(0.09f - eps, 0.04f + eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.04f - eps, 0.01f + eps), 0.01f); - TestSetPacketLossRate(states.get(), I(0.01f - eps, 0.00f ), 0.00f); - // clang-format on -} - -TEST_P(AudioEncoderOpusTest, PacketLossRateLowerBounded) { - test::ScopedFieldTrials override_field_trials( - "WebRTC-Audio-OpusMinPacketLossRate/Enabled-5/"); - auto states = CreateCodec(sample_rate_hz_, 1); - auto I = [](float a, float b) { return IntervalSteps(a, b, 10); }; - constexpr float eps = 1e-8f; - - // clang-format off - TestSetPacketLossRate(states.get(), I(0.00f , 0.01f - eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.01f + eps, 0.06f - eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.06f + eps, 0.11f - eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.11f + eps, 0.22f - eps), 0.10f); - TestSetPacketLossRate(states.get(), I(0.22f + eps, 1.00f ), 0.20f); - - TestSetPacketLossRate(states.get(), I(1.00f , 0.18f + eps), 0.20f); - TestSetPacketLossRate(states.get(), I(0.18f - eps, 0.09f + eps), 0.10f); - TestSetPacketLossRate(states.get(), I(0.09f - eps, 0.04f + eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.04f - eps, 0.01f + eps), 0.05f); - TestSetPacketLossRate(states.get(), I(0.01f - eps, 0.00f ), 0.05f); - // clang-format on -} - TEST_P(AudioEncoderOpusTest, SetReceiverFrameLengthRange) { auto states = CreateCodec(sample_rate_hz_, 2); // Before calling to |SetReceiverFrameLengthRange|, @@ -337,6 +254,8 @@ TEST_P(AudioEncoderOpusTest, TEST_P(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedUplinkBandwidth) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-StableTargetAdaptation/Disabled/"); auto states = CreateCodec(sample_rate_hz_, 2); states->encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -358,6 +277,28 @@ TEST_P(AudioEncoderOpusTest, CheckEncoderRuntimeConfig(states->encoder.get(), config); } +TEST_P(AudioEncoderOpusTest, + InvokeAudioNetworkAdaptorOnReceivedUplinkAllocation) { + auto states = CreateCodec(sample_rate_hz_, 2); + states->encoder->EnableAudioNetworkAdaptor("", nullptr); + + auto config = CreateEncoderRuntimeConfig(); + EXPECT_CALL(*states->mock_audio_network_adaptor, GetEncoderRuntimeConfig()) + .WillOnce(Return(config)); + + BitrateAllocationUpdate update; + update.target_bitrate = DataRate::BitsPerSec(30000); + update.stable_target_bitrate = DataRate::BitsPerSec(20000); + update.bwe_period = TimeDelta::Millis(200); + EXPECT_CALL(*states->mock_audio_network_adaptor, + SetTargetAudioBitrate(update.target_bitrate.bps())); + EXPECT_CALL(*states->mock_audio_network_adaptor, + SetUplinkBandwidth(update.stable_target_bitrate.bps())); + states->encoder->OnReceivedUplinkAllocation(update); + + CheckEncoderRuntimeConfig(states->encoder.get(), config); +} + TEST_P(AudioEncoderOpusTest, InvokeAudioNetworkAdaptorOnReceivedRtt) { auto states = CreateCodec(sample_rate_hz_, 2); states->encoder->EnableAudioNetworkAdaptor("", nullptr); @@ -404,16 +345,21 @@ TEST_P(AudioEncoderOpusTest, // First time, no filtering. states->encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_1); - EXPECT_FLOAT_EQ(0.01f, states->encoder->packet_loss_rate()); + EXPECT_FLOAT_EQ(0.02f, states->encoder->packet_loss_rate()); states->fake_clock->AdvanceTime(TimeDelta::Millis(kSecondSampleTimeMs)); states->encoder->OnReceivedUplinkPacketLossFraction(kPacketLossFraction_2); // Now the output of packet loss fraction smoother should be - // (0.02 + 0.198) / 2 = 0.109, which reach the threshold for the optimized - // packet loss rate to increase to 0.05. If no smoothing has been made, the - // optimized packet loss rate should have been increase to 0.1. - EXPECT_FLOAT_EQ(0.05f, states->encoder->packet_loss_rate()); + // (0.02 + 0.198) / 2 = 0.109. + EXPECT_NEAR(0.109f, states->encoder->packet_loss_rate(), 0.001); +} + +TEST_P(AudioEncoderOpusTest, PacketLossRateUpperBounded) { + auto states = CreateCodec(sample_rate_hz_, 2); + + states->encoder->OnReceivedUplinkPacketLossFraction(0.5); + EXPECT_FLOAT_EQ(0.2f, states->encoder->packet_loss_rate()); } TEST_P(AudioEncoderOpusTest, DoNotInvokeSetTargetBitrateIfOverheadUnknown) { @@ -477,29 +423,6 @@ TEST_P(AudioEncoderOpusTest, BitrateBounded) { EXPECT_EQ(kMaxBitrateBps, states->encoder->GetTargetBitrate()); } -TEST_P(AudioEncoderOpusTest, MinPacketLossRate) { - constexpr float kDefaultMinPacketLossRate = 0.01; - { - test::ScopedFieldTrials override_field_trials( - "WebRTC-Audio-OpusMinPacketLossRate/Enabled/"); - auto states = CreateCodec(sample_rate_hz_, 1); - EXPECT_EQ(kDefaultMinPacketLossRate, states->encoder->packet_loss_rate()); - } - { - test::ScopedFieldTrials override_field_trials( - "WebRTC-Audio-OpusMinPacketLossRate/Enabled-200/"); - auto states = CreateCodec(sample_rate_hz_, 1); - EXPECT_EQ(kDefaultMinPacketLossRate, states->encoder->packet_loss_rate()); - } - { - test::ScopedFieldTrials override_field_trials( - "WebRTC-Audio-OpusMinPacketLossRate/Enabled-50/"); - constexpr float kMinPacketLossRate = 0.5; - auto states = CreateCodec(sample_rate_hz_, 1); - EXPECT_EQ(kMinPacketLossRate, states->encoder->packet_loss_rate()); - } -} - // Verifies that the complexity adaptation in the config works as intended. TEST(AudioEncoderOpusTest, ConfigComplexityAdaptation) { AudioEncoderOpusConfig config; @@ -602,6 +525,8 @@ TEST_P(AudioEncoderOpusTest, EmptyConfigDoesNotAffectEncoderSettings) { } TEST_P(AudioEncoderOpusTest, UpdateUplinkBandwidthInAudioNetworkAdaptor) { + test::ScopedFieldTrials override_field_trials( + "WebRTC-Audio-StableTargetAdaptation/Disabled/"); auto states = CreateCodec(sample_rate_hz_, 2); states->encoder->EnableAudioNetworkAdaptor("", nullptr); const size_t opus_rate_khz = rtc::CheckedDivExact(sample_rate_hz_, 1000); diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.cc index 4bac365a89a..ca39ed82354 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.cc @@ -678,33 +678,7 @@ int WebRtcOpus_FecDurationEst(const uint8_t* payload, return samples; } -// This method is based on Definition of the Opus Audio Codec -// (https://tools.ietf.org/html/rfc6716). Basically, this method is based on -// parsing the LP layer of an Opus packet, particularly the LBRR flag. -int WebRtcOpus_PacketHasFec(const uint8_t* payload, - size_t payload_length_bytes) { - if (payload == NULL || payload_length_bytes == 0) - return 0; - - // In CELT_ONLY mode, packets should not have FEC. - if (payload[0] & 0x80) - return 0; - - // Max number of frames in an Opus packet is 48. - opus_int16 frame_sizes[48]; - const unsigned char* frame_data[48]; - - // Parse packet to get the frames. But we only care about the first frame, - // since we can only decode the FEC from the first one. - if (opus_packet_parse(payload, static_cast<opus_int32>(payload_length_bytes), - NULL, frame_data, frame_sizes, NULL) < 0) { - return 0; - } - - if (frame_sizes[0] <= 1) { - return 0; - } - +int WebRtcOpus_NumSilkFrames(const uint8_t* payload) { // For computing the payload length in ms, the sample rate is not important // since it cancels out. We use 48 kHz, but any valid sample rate would work. int payload_length_ms = @@ -727,10 +701,43 @@ int WebRtcOpus_PacketHasFec(const uint8_t* payload, default: return 0; // It is actually even an invalid packet. } + return silk_frames; +} + +// This method is based on Definition of the Opus Audio Codec +// (https://tools.ietf.org/html/rfc6716). Basically, this method is based on +// parsing the LP layer of an Opus packet, particularly the LBRR flag. +int WebRtcOpus_PacketHasFec(const uint8_t* payload, + size_t payload_length_bytes) { + if (payload == NULL || payload_length_bytes == 0) + return 0; + + // In CELT_ONLY mode, packets should not have FEC. + if (payload[0] & 0x80) + return 0; + + int silk_frames = WebRtcOpus_NumSilkFrames(payload); + if (silk_frames == 0) + return 0; // Not valid. const int channels = opus_packet_get_nb_channels(payload); RTC_DCHECK(channels == 1 || channels == 2); + // Max number of frames in an Opus packet is 48. + opus_int16 frame_sizes[48]; + const unsigned char* frame_data[48]; + + // Parse packet to get the frames. But we only care about the first frame, + // since we can only decode the FEC from the first one. + if (opus_packet_parse(payload, static_cast<opus_int32>(payload_length_bytes), + NULL, frame_data, frame_sizes, NULL) < 0) { + return 0; + } + + if (frame_sizes[0] < 1) { + return 0; + } + // A frame starts with the LP layer. The LP layer begins with two to eight // header bits.These consist of one VAD bit per SILK frame (up to 3), // followed by a single flag indicating the presence of LBRR frames. @@ -748,3 +755,45 @@ int WebRtcOpus_PacketHasFec(const uint8_t* payload, return 0; } + +int WebRtcOpus_PacketHasVoiceActivity(const uint8_t* payload, + size_t payload_length_bytes) { + if (payload == NULL || payload_length_bytes == 0) + return 0; + + // In CELT_ONLY mode we can not determine whether there is VAD. + if (payload[0] & 0x80) + return -1; + + int silk_frames = WebRtcOpus_NumSilkFrames(payload); + if (silk_frames == 0) + return -1; + + const int channels = opus_packet_get_nb_channels(payload); + RTC_DCHECK(channels == 1 || channels == 2); + + // Max number of frames in an Opus packet is 48. + opus_int16 frame_sizes[48]; + const unsigned char* frame_data[48]; + + // Parse packet to get the frames. + int frames = + opus_packet_parse(payload, static_cast<opus_int32>(payload_length_bytes), + NULL, frame_data, frame_sizes, NULL); + if (frames < 0) + return -1; + + // Iterate over all Opus frames which may contain multiple SILK frames. + for (int frame = 0; frame < frames; frame++) { + if (frame_sizes[frame] < 1) { + continue; + } + if (frame_data[frame][0] >> (8 - silk_frames)) + return 1; + if (channels == 2 && + (frame_data[frame][0] << (silk_frames + 1)) >> (8 - silk_frames)) + return 1; + } + + return 0; +} diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.h b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.h index e8de9730109..2a3ceaa7d31 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.h +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_interface.h @@ -510,6 +510,22 @@ int WebRtcOpus_FecDurationEst(const uint8_t* payload, int WebRtcOpus_PacketHasFec(const uint8_t* payload, size_t payload_length_bytes); +/**************************************************************************** + * WebRtcOpus_PacketHasVoiceActivity(...) + * + * This function returns the SILK VAD information encoded in the opus packet. + * For CELT-only packets that do not have VAD information, it returns -1. + * Input: + * - payload : Encoded data pointer + * - payload_length_bytes : Bytes of encoded data + * + * Return value : 0 - no frame had the VAD flag set. + * 1 - at least one frame had the VAD flag set. + * -1 - VAD status could not be determined. + */ +int WebRtcOpus_PacketHasVoiceActivity(const uint8_t* payload, + size_t payload_length_bytes); + #ifdef __cplusplus } // extern "C" #endif diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc index 3407d7d3cf9..80cab501372 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/opus/opus_unittest.cc @@ -949,4 +949,30 @@ TEST_P(OpusTest, OpusDecodeRepacketized) { EXPECT_EQ(0, WebRtcOpus_DecoderFree(opus_decoder_)); } +TEST(OpusVadTest, CeltUnknownStatus) { + const uint8_t celt[] = {0x80}; + EXPECT_EQ(WebRtcOpus_PacketHasVoiceActivity(celt, 1), -1); +} + +TEST(OpusVadTest, Mono20msVadSet) { + uint8_t silk20msMonoVad[] = {0x78, 0x80}; + EXPECT_TRUE(WebRtcOpus_PacketHasVoiceActivity(silk20msMonoVad, 2)); +} + +TEST(OpusVadTest, Mono20MsVadUnset) { + uint8_t silk20msMonoSilence[] = {0x78, 0x00}; + EXPECT_FALSE(WebRtcOpus_PacketHasVoiceActivity(silk20msMonoSilence, 2)); +} + +TEST(OpusVadTest, Stereo20MsVadOnSideChannel) { + uint8_t silk20msStereoVadSideChannel[] = {0x78 | 0x04, 0x20}; + EXPECT_TRUE( + WebRtcOpus_PacketHasVoiceActivity(silk20msStereoVadSideChannel, 2)); +} + +TEST(OpusVadTest, TwoOpusMonoFramesVadOnSecond) { + uint8_t twoMonoFrames[] = {0x78 | 0x1, 0x00, 0x80}; + EXPECT_TRUE(WebRtcOpus_PacketHasVoiceActivity(twoMonoFrames, 3)); +} + } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc index e75806af104..8d028c9b9ad 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red.cc @@ -15,6 +15,7 @@ #include <utility> #include <vector> +#include "rtc_base/byte_order.h" #include "rtc_base/checks.h" namespace webrtc { @@ -59,32 +60,62 @@ AudioEncoder::EncodedInfo AudioEncoderCopyRed::EncodeImpl( uint32_t rtp_timestamp, rtc::ArrayView<const int16_t> audio, rtc::Buffer* encoded) { - const size_t primary_offset = encoded->size(); + // Allocate room for RFC 2198 header if there is redundant data. + // Otherwise this will send the primary payload type without + // wrapping in RED. + const size_t header_length_bytes = secondary_info_.encoded_bytes > 0 ? 5 : 0; + size_t secondary_length_bytes = 0; + + if (secondary_info_.encoded_bytes > 0) { + encoded->SetSize(header_length_bytes); + encoded->AppendData(secondary_encoded_); + secondary_length_bytes = secondary_info_.encoded_bytes; + } EncodedInfo info = speech_encoder_->Encode(rtp_timestamp, audio, encoded); + if (info.encoded_bytes == 0) { + encoded->Clear(); + return info; + } + + // Actually construct the RFC 2198 header. + if (secondary_info_.encoded_bytes > 0) { + const uint32_t timestamp_delta = + info.encoded_timestamp - secondary_info_.encoded_timestamp; + + encoded->data()[0] = secondary_info_.payload_type | 0x80; + RTC_DCHECK_LT(secondary_info_.encoded_bytes, 1 << 10); + rtc::SetBE16(static_cast<uint8_t*>(encoded->data()) + 1, + (timestamp_delta << 2) | (secondary_info_.encoded_bytes >> 8)); + encoded->data()[3] = secondary_info_.encoded_bytes & 0xff; + encoded->data()[4] = info.payload_type; + } + RTC_CHECK(info.redundant.empty()) << "Cannot use nested redundant encoders."; - RTC_DCHECK_EQ(encoded->size() - primary_offset, info.encoded_bytes); - - if (info.encoded_bytes > 0) { - // |info| will be implicitly cast to an EncodedInfoLeaf struct, effectively - // discarding the (empty) vector of redundant information. This is - // intentional. - info.redundant.push_back(info); - RTC_DCHECK_EQ(info.redundant.size(), 1); - if (secondary_info_.encoded_bytes > 0) { - encoded->AppendData(secondary_encoded_); - info.redundant.push_back(secondary_info_); - RTC_DCHECK_EQ(info.redundant.size(), 2); - } - // Save primary to secondary. - secondary_encoded_.SetData(encoded->data() + primary_offset, - info.encoded_bytes); - secondary_info_ = info; - RTC_DCHECK_EQ(info.speech, info.redundant[0].speech); + RTC_DCHECK_EQ(encoded->size() - header_length_bytes - secondary_length_bytes, + info.encoded_bytes); + + // |info| will be implicitly cast to an EncodedInfoLeaf struct, effectively + // discarding the (empty) vector of redundant information. This is + // intentional. + info.redundant.push_back(info); + RTC_DCHECK_EQ(info.redundant.size(), 1); + if (secondary_info_.encoded_bytes > 0) { + info.redundant.push_back(secondary_info_); + RTC_DCHECK_EQ(info.redundant.size(), 2); } + // Save primary to secondary. + secondary_encoded_.SetData( + &encoded->data()[header_length_bytes + secondary_info_.encoded_bytes], + info.encoded_bytes); + secondary_info_ = info; + RTC_DCHECK_EQ(info.speech, info.redundant[0].speech); + // Update main EncodedInfo. - info.payload_type = red_payload_type_; - info.encoded_bytes = 0; + if (header_length_bytes > 0) { + info.payload_type = red_payload_type_; + } + info.encoded_bytes = header_length_bytes; for (std::vector<EncodedInfoLeaf>::const_iterator it = info.redundant.begin(); it != info.redundant.end(); ++it) { info.encoded_bytes += it->encoded_bytes; diff --git a/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc index e20515a1650..720acb4f879 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/codecs/red/audio_encoder_copy_red_unittest.cc @@ -139,6 +139,7 @@ TEST_F(AudioEncoderCopyRedTest, CheckImmediateEncode) { // new data, even if the RED codec is loaded with a secondary encoding. TEST_F(AudioEncoderCopyRedTest, CheckNoOutput) { static const size_t kEncodedSize = 17; + static const size_t kHeaderLenBytes = 5; { InSequence s; EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) @@ -160,7 +161,7 @@ TEST_F(AudioEncoderCopyRedTest, CheckNoOutput) { // Final call to the speech encoder will produce output. Encode(); - EXPECT_EQ(2 * kEncodedSize, encoded_info_.encoded_bytes); + EXPECT_EQ(2 * kEncodedSize + kHeaderLenBytes, encoded_info_.encoded_bytes); ASSERT_EQ(2u, encoded_info_.redundant.size()); } @@ -187,7 +188,7 @@ TEST_F(AudioEncoderCopyRedTest, CheckPayloadSizes) { ASSERT_EQ(2u, encoded_info_.redundant.size()); EXPECT_EQ(i, encoded_info_.redundant[0].encoded_bytes); EXPECT_EQ(i - 1, encoded_info_.redundant[1].encoded_bytes); - EXPECT_EQ(i + i - 1, encoded_info_.encoded_bytes); + EXPECT_EQ(5 + i + i - 1, encoded_info_.encoded_bytes); } } @@ -224,6 +225,7 @@ TEST_F(AudioEncoderCopyRedTest, CheckPayloads) { // Let the mock encoder write payloads with increasing values. The first // payload will have values 0, 1, 2, ..., kPayloadLenBytes - 1. static const size_t kPayloadLenBytes = 5; + static const size_t kHeaderLenBytes = 5; uint8_t payload[kPayloadLenBytes]; for (uint8_t i = 0; i < kPayloadLenBytes; ++i) { payload[i] = i; @@ -239,7 +241,7 @@ TEST_F(AudioEncoderCopyRedTest, CheckPayloads) { EXPECT_EQ(i, encoded_.data()[i]); } - for (int j = 0; j < 5; ++j) { + for (int j = 0; j < 1; ++j) { // Increment all values of the payload by 10. for (size_t i = 0; i < kPayloadLenBytes; ++i) payload[i] += 10; @@ -249,16 +251,17 @@ TEST_F(AudioEncoderCopyRedTest, CheckPayloads) { EXPECT_EQ(kPayloadLenBytes, encoded_info_.redundant[0].encoded_bytes); EXPECT_EQ(kPayloadLenBytes, encoded_info_.redundant[1].encoded_bytes); for (size_t i = 0; i < kPayloadLenBytes; ++i) { - // Check primary payload. - EXPECT_EQ((j + 1) * 10 + i, encoded_.data()[i]); // Check secondary payload. - EXPECT_EQ(j * 10 + i, encoded_.data()[i + kPayloadLenBytes]); + EXPECT_EQ(j * 10 + i, encoded_.data()[kHeaderLenBytes + i]); + + // Check primary payload. + EXPECT_EQ((j + 1) * 10 + i, + encoded_.data()[kHeaderLenBytes + i + kPayloadLenBytes]); } } } // Checks correct propagation of payload type. -// Checks that the correct timestamps are returned. TEST_F(AudioEncoderCopyRedTest, CheckPayloadType) { const int primary_payload_type = red_payload_type_ + 1; AudioEncoder::EncodedInfo info; @@ -272,7 +275,7 @@ TEST_F(AudioEncoderCopyRedTest, CheckPayloadType) { Encode(); ASSERT_EQ(1u, encoded_info_.redundant.size()); EXPECT_EQ(primary_payload_type, encoded_info_.redundant[0].payload_type); - EXPECT_EQ(red_payload_type_, encoded_info_.payload_type); + EXPECT_EQ(primary_payload_type, encoded_info_.payload_type); const int secondary_payload_type = red_payload_type_ + 2; info.payload_type = secondary_payload_type; @@ -286,6 +289,36 @@ TEST_F(AudioEncoderCopyRedTest, CheckPayloadType) { EXPECT_EQ(red_payload_type_, encoded_info_.payload_type); } +TEST_F(AudioEncoderCopyRedTest, CheckRFC2198Header) { + const int primary_payload_type = red_payload_type_ + 1; + AudioEncoder::EncodedInfo info; + info.encoded_bytes = 10; + info.encoded_timestamp = timestamp_; + info.payload_type = primary_payload_type; + + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); + info.encoded_timestamp = timestamp_; // update timestamp. + EXPECT_CALL(*mock_encoder_, EncodeImpl(_, _, _)) + .WillOnce(Invoke(MockAudioEncoder::FakeEncoding(info))); + Encode(); // Second call will produce a redundant encoding. + + EXPECT_EQ(encoded_.size(), + 5u + 2 * 10u); // header size + two encoded payloads. + EXPECT_EQ(encoded_[0], primary_payload_type | 0x80); + + uint32_t timestamp_delta = encoded_info_.encoded_timestamp - + encoded_info_.redundant[1].encoded_timestamp; + // Timestamp delta is encoded as a 14 bit value. + EXPECT_EQ(encoded_[1], timestamp_delta >> 6); + EXPECT_EQ(static_cast<uint8_t>(encoded_[2] >> 2), timestamp_delta & 0x3f); + // Redundant length is encoded as 10 bit value. + EXPECT_EQ(encoded_[2] & 0x3u, encoded_info_.redundant[1].encoded_bytes >> 8); + EXPECT_EQ(encoded_[3], encoded_info_.redundant[1].encoded_bytes & 0xff); + EXPECT_EQ(encoded_[4], primary_payload_type); +} + #if GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // This test fixture tests various error conditions that makes the diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc index 836c49c12fa..d1e1ec1e307 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/audio_decoder_unittest.cc @@ -536,7 +536,11 @@ TEST_F(AudioDecoderIsacFloatTest, EncodeDecode) { } TEST_F(AudioDecoderIsacFloatTest, SetTargetBitrate) { - TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 32000); + EXPECT_EQ(10000, SetAndGetTargetBitrate(audio_encoder_.get(), 9999)); + EXPECT_EQ(10000, SetAndGetTargetBitrate(audio_encoder_.get(), 10000)); + EXPECT_EQ(23456, SetAndGetTargetBitrate(audio_encoder_.get(), 23456)); + EXPECT_EQ(32000, SetAndGetTargetBitrate(audio_encoder_.get(), 32000)); + EXPECT_EQ(32000, SetAndGetTargetBitrate(audio_encoder_.get(), 32001)); } TEST_F(AudioDecoderIsacSwbTest, EncodeDecode) { @@ -549,7 +553,11 @@ TEST_F(AudioDecoderIsacSwbTest, EncodeDecode) { } TEST_F(AudioDecoderIsacSwbTest, SetTargetBitrate) { - TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 32000); + EXPECT_EQ(10000, SetAndGetTargetBitrate(audio_encoder_.get(), 9999)); + EXPECT_EQ(10000, SetAndGetTargetBitrate(audio_encoder_.get(), 10000)); + EXPECT_EQ(23456, SetAndGetTargetBitrate(audio_encoder_.get(), 23456)); + EXPECT_EQ(56000, SetAndGetTargetBitrate(audio_encoder_.get(), 56000)); + EXPECT_EQ(56000, SetAndGetTargetBitrate(audio_encoder_.get(), 56001)); } TEST_F(AudioDecoderIsacFixTest, EncodeDecode) { @@ -569,7 +577,11 @@ TEST_F(AudioDecoderIsacFixTest, EncodeDecode) { } TEST_F(AudioDecoderIsacFixTest, SetTargetBitrate) { - TestSetAndGetTargetBitratesWithFixedCodec(audio_encoder_.get(), 32000); + EXPECT_EQ(10000, SetAndGetTargetBitrate(audio_encoder_.get(), 9999)); + EXPECT_EQ(10000, SetAndGetTargetBitrate(audio_encoder_.get(), 10000)); + EXPECT_EQ(23456, SetAndGetTargetBitrate(audio_encoder_.get(), 23456)); + EXPECT_EQ(32000, SetAndGetTargetBitrate(audio_encoder_.get(), 32000)); + EXPECT_EQ(32000, SetAndGetTargetBitrate(audio_encoder_.get(), 32001)); } TEST_F(AudioDecoderG722Test, EncodeDecode) { diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h index d83dc7f62ca..b8dc031fa4b 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_decoder_database.h @@ -23,22 +23,28 @@ class MockDecoderDatabase : public DecoderDatabase { explicit MockDecoderDatabase( rtc::scoped_refptr<AudioDecoderFactory> factory = nullptr) : DecoderDatabase(factory, absl::nullopt) {} - virtual ~MockDecoderDatabase() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_CONST_METHOD0(Empty, bool()); - MOCK_CONST_METHOD0(Size, int()); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD2(RegisterPayload, - int(int rtp_payload_type, const SdpAudioFormat& audio_format)); - MOCK_METHOD1(Remove, int(uint8_t rtp_payload_type)); - MOCK_METHOD0(RemoveAll, void()); - MOCK_CONST_METHOD1(GetDecoderInfo, - const DecoderInfo*(uint8_t rtp_payload_type)); - MOCK_METHOD2(SetActiveDecoder, - int(uint8_t rtp_payload_type, bool* new_decoder)); - MOCK_CONST_METHOD0(GetActiveDecoder, AudioDecoder*()); - MOCK_METHOD1(SetActiveCngDecoder, int(uint8_t rtp_payload_type)); - MOCK_CONST_METHOD0(GetActiveCngDecoder, ComfortNoiseDecoder*()); + ~MockDecoderDatabase() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(bool, Empty, (), (const, override)); + MOCK_METHOD(int, Size, (), (const, override)); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(int, + RegisterPayload, + (int rtp_payload_type, const SdpAudioFormat& audio_format), + (override)); + MOCK_METHOD(int, Remove, (uint8_t rtp_payload_type), (override)); + MOCK_METHOD(void, RemoveAll, (), (override)); + MOCK_METHOD(const DecoderInfo*, + GetDecoderInfo, + (uint8_t rtp_payload_type), + (const, override)); + MOCK_METHOD(int, + SetActiveDecoder, + (uint8_t rtp_payload_type, bool* new_decoder), + (override)); + MOCK_METHOD(AudioDecoder*, GetActiveDecoder, (), (const, override)); + MOCK_METHOD(int, SetActiveCngDecoder, (uint8_t rtp_payload_type), (override)); + MOCK_METHOD(ComfortNoiseDecoder*, GetActiveCngDecoder, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h index d9fe5d4492d..c60c56d36b3 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_buffer.h @@ -19,13 +19,16 @@ namespace webrtc { class MockDtmfBuffer : public DtmfBuffer { public: MockDtmfBuffer(int fs) : DtmfBuffer(fs) {} - virtual ~MockDtmfBuffer() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD0(Flush, void()); - MOCK_METHOD1(InsertEvent, int(const DtmfEvent& event)); - MOCK_METHOD2(GetEvent, bool(uint32_t current_timestamp, DtmfEvent* event)); - MOCK_CONST_METHOD0(Length, size_t()); - MOCK_CONST_METHOD0(Empty, bool()); + ~MockDtmfBuffer() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Flush, (), (override)); + MOCK_METHOD(int, InsertEvent, (const DtmfEvent& event), (override)); + MOCK_METHOD(bool, + GetEvent, + (uint32_t current_timestamp, DtmfEvent* event), + (override)); + MOCK_METHOD(size_t, Length, (), (const, override)); + MOCK_METHOD(bool, Empty, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h index eea8bee1c2c..60de167c29b 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_dtmf_tone_generator.h @@ -18,12 +18,15 @@ namespace webrtc { class MockDtmfToneGenerator : public DtmfToneGenerator { public: - virtual ~MockDtmfToneGenerator() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD3(Init, int(int fs, int event, int attenuation)); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD2(Generate, int(size_t num_samples, AudioMultiVector* output)); - MOCK_CONST_METHOD0(initialized, bool()); + ~MockDtmfToneGenerator() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(int, Init, (int fs, int event, int attenuation), (override)); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(int, + Generate, + (size_t num_samples, AudioMultiVector* output), + (override)); + MOCK_METHOD(bool, initialized, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_expand.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_expand.h index 286325b841e..9d667790216 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_expand.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_expand.h @@ -30,13 +30,13 @@ class MockExpand : public Expand { statistics, fs, num_channels) {} - virtual ~MockExpand() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD1(Process, int(AudioMultiVector* output)); - MOCK_METHOD0(SetParametersForNormalAfterExpand, void()); - MOCK_METHOD0(SetParametersForMergeAfterExpand, void()); - MOCK_CONST_METHOD0(overlap_length, size_t()); + ~MockExpand() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(int, Process, (AudioMultiVector * output), (override)); + MOCK_METHOD(void, SetParametersForNormalAfterExpand, (), (override)); + MOCK_METHOD(void, SetParametersForMergeAfterExpand, (), (override)); + MOCK_METHOD(size_t, overlap_length, (), (const, override)); }; } // namespace webrtc @@ -45,13 +45,15 @@ namespace webrtc { class MockExpandFactory : public ExpandFactory { public: - MOCK_CONST_METHOD6(Create, - Expand*(BackgroundNoise* background_noise, - SyncBuffer* sync_buffer, - RandomVector* random_vector, - StatisticsCalculator* statistics, - int fs, - size_t num_channels)); + MOCK_METHOD(Expand*, + Create, + (BackgroundNoise * background_noise, + SyncBuffer* sync_buffer, + RandomVector* random_vector, + StatisticsCalculator* statistics, + int fs, + size_t num_channels), + (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_histogram.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_histogram.h index 91ae18f5e8a..03abbc1d4b0 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_histogram.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_histogram.h @@ -22,8 +22,8 @@ class MockHistogram : public Histogram { : Histogram(num_buckets, forget_factor) {} virtual ~MockHistogram() {} - MOCK_METHOD1(Add, void(int)); - MOCK_METHOD1(Quantile, int(int)); + MOCK_METHOD(void, Add, (int), (override)); + MOCK_METHOD(int, Quantile, (int), (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h index d1008c8a30a..b7df85fb205 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_neteq_controller.h @@ -19,46 +19,45 @@ namespace webrtc { class MockNetEqController : public NetEqController { public: MockNetEqController() = default; - virtual ~MockNetEqController() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD0(SoftReset, void()); - MOCK_METHOD2(GetDecision, - NetEq::Operation(const NetEqStatus& neteq_status, - bool* reset_decoder)); - MOCK_METHOD6(Update, - void(uint16_t sequence_number, - uint32_t timestamp, - uint32_t last_played_out_timestamp, - bool new_codec, - bool cng_or_dtmf, - size_t packet_length_samples)); - MOCK_METHOD0(RegisterEmptyPacket, void()); - MOCK_METHOD2(SetSampleRate, void(int fs_hz, size_t output_size_samples)); - MOCK_METHOD1(SetMaximumDelay, bool(int delay_ms)); - MOCK_METHOD1(SetMinimumDelay, bool(int delay_ms)); - MOCK_METHOD1(SetBaseMinimumDelay, bool(int delay_ms)); - MOCK_CONST_METHOD0(GetBaseMinimumDelay, int()); - MOCK_CONST_METHOD0(CngRfc3389On, bool()); - MOCK_CONST_METHOD0(CngOff, bool()); - MOCK_METHOD0(SetCngOff, void()); - MOCK_METHOD1(ExpandDecision, void(NetEq::Operation operation)); - MOCK_METHOD1(AddSampleMemory, void(int32_t value)); - MOCK_METHOD0(TargetLevelMs, int()); - MOCK_METHOD6(PacketArrived, - absl::optional<int>(bool last_cng_or_dtmf, - size_t packet_length_samples, - bool should_update_stats, - uint16_t main_sequence_number, - uint32_t main_timestamp, - int fs_hz)); - MOCK_CONST_METHOD0(PeakFound, bool()); - MOCK_CONST_METHOD0(GetFilteredBufferLevel, int()); - MOCK_METHOD1(set_sample_memory, void(int32_t value)); - MOCK_CONST_METHOD0(noise_fast_forward, size_t()); - MOCK_CONST_METHOD0(packet_length_samples, size_t()); - MOCK_METHOD1(set_packet_length_samples, void(size_t value)); - MOCK_METHOD1(set_prev_time_scale, void(bool value)); + ~MockNetEqController() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(void, SoftReset, (), (override)); + MOCK_METHOD(NetEq::Operation, + GetDecision, + (const NetEqStatus& neteq_status, bool* reset_decoder), + (override)); + MOCK_METHOD(void, RegisterEmptyPacket, (), (override)); + MOCK_METHOD(void, + SetSampleRate, + (int fs_hz, size_t output_size_samples), + (override)); + MOCK_METHOD(bool, SetMaximumDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, SetMinimumDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, SetBaseMinimumDelay, (int delay_ms), (override)); + MOCK_METHOD(int, GetBaseMinimumDelay, (), (const, override)); + MOCK_METHOD(bool, CngRfc3389On, (), (const, override)); + MOCK_METHOD(bool, CngOff, (), (const, override)); + MOCK_METHOD(void, SetCngOff, (), (override)); + MOCK_METHOD(void, ExpandDecision, (NetEq::Operation operation), (override)); + MOCK_METHOD(void, AddSampleMemory, (int32_t value), (override)); + MOCK_METHOD(int, TargetLevelMs, (), (override)); + MOCK_METHOD(absl::optional<int>, + PacketArrived, + (bool last_cng_or_dtmf, + size_t packet_length_samples, + bool should_update_stats, + uint16_t main_sequence_number, + uint32_t main_timestamp, + int fs_hz), + (override)); + MOCK_METHOD(bool, PeakFound, (), (const, override)); + MOCK_METHOD(int, GetFilteredBufferLevel, (), (const, override)); + MOCK_METHOD(void, set_sample_memory, (int32_t value), (override)); + MOCK_METHOD(size_t, noise_fast_forward, (), (const, override)); + MOCK_METHOD(size_t, packet_length_samples, (), (const, override)); + MOCK_METHOD(void, set_packet_length_samples, (size_t value), (override)); + MOCK_METHOD(void, set_prev_time_scale, (bool value), (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h index 7efeb15e473..e466ea6c8bb 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_packet_buffer.h @@ -20,39 +20,47 @@ class MockPacketBuffer : public PacketBuffer { public: MockPacketBuffer(size_t max_number_of_packets, const TickTimer* tick_timer) : PacketBuffer(max_number_of_packets, tick_timer) {} - virtual ~MockPacketBuffer() { Die(); } - MOCK_METHOD0(Die, void()); - MOCK_METHOD0(Flush, void()); - MOCK_CONST_METHOD0(Empty, bool()); - int InsertPacket(Packet&& packet, StatisticsCalculator* stats) { - return InsertPacketWrapped(&packet, stats); - } - // Since gtest does not properly support move-only types, InsertPacket is - // implemented as a wrapper. You'll have to implement InsertPacketWrapped - // instead and move from |*packet|. - MOCK_METHOD2(InsertPacketWrapped, - int(Packet* packet, StatisticsCalculator* stats)); - MOCK_METHOD5(InsertPacketList, - int(PacketList* packet_list, - const DecoderDatabase& decoder_database, - absl::optional<uint8_t>* current_rtp_payload_type, - absl::optional<uint8_t>* current_cng_rtp_payload_type, - StatisticsCalculator* stats)); - MOCK_CONST_METHOD1(NextTimestamp, int(uint32_t* next_timestamp)); - MOCK_CONST_METHOD2(NextHigherTimestamp, - int(uint32_t timestamp, uint32_t* next_timestamp)); - MOCK_CONST_METHOD0(PeekNextPacket, const Packet*()); - MOCK_METHOD0(GetNextPacket, absl::optional<Packet>()); - MOCK_METHOD1(DiscardNextPacket, int(StatisticsCalculator* stats)); - MOCK_METHOD3(DiscardOldPackets, - void(uint32_t timestamp_limit, - uint32_t horizon_samples, - StatisticsCalculator* stats)); - MOCK_METHOD2(DiscardAllOldPackets, - void(uint32_t timestamp_limit, StatisticsCalculator* stats)); - MOCK_CONST_METHOD0(NumPacketsInBuffer, size_t()); - MOCK_METHOD1(IncrementWaitingTimes, void(int)); - MOCK_CONST_METHOD0(current_memory_bytes, int()); + ~MockPacketBuffer() override { Die(); } + MOCK_METHOD(void, Die, ()); + MOCK_METHOD(void, Flush, (), (override)); + MOCK_METHOD(bool, Empty, (), (const, override)); + MOCK_METHOD(int, + InsertPacket, + (Packet && packet, StatisticsCalculator* stats), + (override)); + MOCK_METHOD(int, + InsertPacketList, + (PacketList * packet_list, + const DecoderDatabase& decoder_database, + absl::optional<uint8_t>* current_rtp_payload_type, + absl::optional<uint8_t>* current_cng_rtp_payload_type, + StatisticsCalculator* stats), + (override)); + MOCK_METHOD(int, + NextTimestamp, + (uint32_t * next_timestamp), + (const, override)); + MOCK_METHOD(int, + NextHigherTimestamp, + (uint32_t timestamp, uint32_t* next_timestamp), + (const, override)); + MOCK_METHOD(const Packet*, PeekNextPacket, (), (const, override)); + MOCK_METHOD(absl::optional<Packet>, GetNextPacket, (), (override)); + MOCK_METHOD(int, + DiscardNextPacket, + (StatisticsCalculator * stats), + (override)); + MOCK_METHOD(void, + DiscardOldPackets, + (uint32_t timestamp_limit, + uint32_t horizon_samples, + StatisticsCalculator* stats), + (override)); + MOCK_METHOD(void, + DiscardAllOldPackets, + (uint32_t timestamp_limit, StatisticsCalculator* stats), + (override)); + MOCK_METHOD(size_t, NumPacketsInBuffer, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h index 68fd3566c92..9daf571a804 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_red_payload_splitter.h @@ -18,10 +18,12 @@ namespace webrtc { class MockRedPayloadSplitter : public RedPayloadSplitter { public: - MOCK_METHOD1(SplitRed, bool(PacketList* packet_list)); - MOCK_METHOD2(CheckRedPayloads, - void(PacketList* packet_list, - const DecoderDatabase& decoder_database)); + MOCK_METHOD(bool, SplitRed, (PacketList * packet_list), (override)); + MOCK_METHOD(void, + CheckRedPayloads, + (PacketList * packet_list, + const DecoderDatabase& decoder_database), + (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h index 086c7c55644..f8812478d6d 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/mock/mock_statistics_calculator.h @@ -18,9 +18,12 @@ namespace webrtc { class MockStatisticsCalculator : public StatisticsCalculator { public: - MOCK_METHOD1(PacketsDiscarded, void(size_t num_packets)); - MOCK_METHOD1(SecondaryPacketsDiscarded, void(size_t num_packets)); - MOCK_METHOD1(RelativePacketArrivalDelay, void(size_t delay_ms)); + MOCK_METHOD(void, PacketsDiscarded, (size_t num_packets), (override)); + MOCK_METHOD(void, + SecondaryPacketsDiscarded, + (size_t num_packets), + (override)); + MOCK_METHOD(void, RelativePacketArrivalDelay, (size_t delay_ms), (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.cc index 0b7510d3418..f1cd8015e6f 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.cc @@ -51,6 +51,7 @@ #include "rtc_base/strings/audio_format_to_string.h" #include "rtc_base/trace_event.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { @@ -73,6 +74,24 @@ std::unique_ptr<NetEqController> CreateNetEqController( return controller_factory.CreateNetEqController(config); } +int GetDelayChainLengthMs(int config_extra_delay_ms) { + constexpr char kExtraDelayFieldTrial[] = "WebRTC-Audio-NetEqExtraDelay"; + if (webrtc::field_trial::IsEnabled(kExtraDelayFieldTrial)) { + const auto field_trial_string = + webrtc::field_trial::FindFullName(kExtraDelayFieldTrial); + int extra_delay_ms = -1; + if (sscanf(field_trial_string.c_str(), "Enabled-%d", &extra_delay_ms) == + 1 && + extra_delay_ms >= 0 && extra_delay_ms <= 2000) { + RTC_LOG(LS_INFO) << "Delay chain length set to " << extra_delay_ms + << " ms in field trial"; + return (extra_delay_ms / 10) * 10; // Rounding down to multiple of 10. + } + } + // Field trial not set, or invalid value read. Use value from config. + return config_extra_delay_ms; +} + } // namespace NetEqImpl::Dependencies::Dependencies( @@ -140,7 +159,10 @@ NetEqImpl::NetEqImpl(const NetEq::Config& config, 10, // Report once every 10 s. tick_timer_.get()), no_time_stretching_(config.for_test_no_time_stretching), - enable_rtx_handling_(config.enable_rtx_handling) { + enable_rtx_handling_(config.enable_rtx_handling), + output_delay_chain_ms_( + GetDelayChainLengthMs(config.extra_output_delay_ms)), + output_delay_chain_(rtc::CheckedDivExact(output_delay_chain_ms_, 10)) { RTC_LOG(LS_INFO) << "NetEq config: " << config.ToString(); int fs = config.sample_rate_hz; if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) { @@ -255,6 +277,25 @@ int NetEqImpl::GetAudio(AudioFrame* audio_frame, last_output_sample_rate_hz_ == 32000 || last_output_sample_rate_hz_ == 48000) << "Unexpected sample rate " << last_output_sample_rate_hz_; + + if (!output_delay_chain_.empty()) { + if (output_delay_chain_empty_) { + for (auto& f : output_delay_chain_) { + f.CopyFrom(*audio_frame); + } + output_delay_chain_empty_ = false; + delayed_last_output_sample_rate_hz_ = last_output_sample_rate_hz_; + } else { + RTC_DCHECK_GE(output_delay_chain_ix_, 0); + RTC_DCHECK_LT(output_delay_chain_ix_, output_delay_chain_.size()); + swap(output_delay_chain_[output_delay_chain_ix_], *audio_frame); + *muted = audio_frame->muted(); + output_delay_chain_ix_ = + (output_delay_chain_ix_ + 1) % output_delay_chain_.size(); + delayed_last_output_sample_rate_hz_ = audio_frame->sample_rate_hz(); + } + } + return kOK; } @@ -297,7 +338,8 @@ bool NetEqImpl::SetMinimumDelay(int delay_ms) { rtc::CritScope lock(&crit_sect_); if (delay_ms >= 0 && delay_ms <= 10000) { assert(controller_.get()); - return controller_->SetMinimumDelay(delay_ms); + return controller_->SetMinimumDelay( + std::max(delay_ms - output_delay_chain_ms_, 0)); } return false; } @@ -306,7 +348,8 @@ bool NetEqImpl::SetMaximumDelay(int delay_ms) { rtc::CritScope lock(&crit_sect_); if (delay_ms >= 0 && delay_ms <= 10000) { assert(controller_.get()); - return controller_->SetMaximumDelay(delay_ms); + return controller_->SetMaximumDelay( + std::max(delay_ms - output_delay_chain_ms_, 0)); } return false; } @@ -327,7 +370,7 @@ int NetEqImpl::GetBaseMinimumDelayMs() const { int NetEqImpl::TargetDelayMs() const { rtc::CritScope lock(&crit_sect_); RTC_DCHECK(controller_.get()); - return controller_->TargetLevelMs(); + return controller_->TargetLevelMs() + output_delay_chain_ms_; } int NetEqImpl::FilteredCurrentDelayMs() const { @@ -337,7 +380,8 @@ int NetEqImpl::FilteredCurrentDelayMs() const { const int delay_samples = controller_->GetFilteredBufferLevel() + sync_buffer_->FutureLength(); // The division below will truncate. The return value is in ms. - return delay_samples / rtc::CheckedDivExact(fs_hz_, 1000); + return delay_samples / rtc::CheckedDivExact(fs_hz_, 1000) + + output_delay_chain_ms_; } int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) { @@ -351,6 +395,13 @@ int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) { stats->jitter_peaks_found = controller_->PeakFound(); stats_->GetNetworkStatistics(fs_hz_, total_samples_in_buffers, decoder_frame_length_, stats); + // Compensate for output delay chain. + stats->current_buffer_size_ms += output_delay_chain_ms_; + stats->preferred_buffer_size_ms += output_delay_chain_ms_; + stats->mean_waiting_time_ms += output_delay_chain_ms_; + stats->median_waiting_time_ms += output_delay_chain_ms_; + stats->min_waiting_time_ms += output_delay_chain_ms_; + stats->max_waiting_time_ms += output_delay_chain_ms_; return 0; } @@ -394,12 +445,19 @@ absl::optional<uint32_t> NetEqImpl::GetPlayoutTimestamp() const { // which is indicated by returning an empty value. return absl::nullopt; } - return timestamp_scaler_->ToExternal(playout_timestamp_); + size_t sum_samples_in_output_delay_chain = 0; + for (const auto& audio_frame : output_delay_chain_) { + sum_samples_in_output_delay_chain += audio_frame.samples_per_channel(); + } + return timestamp_scaler_->ToExternal( + playout_timestamp_ - + static_cast<uint32_t>(sum_samples_in_output_delay_chain)); } int NetEqImpl::last_output_sample_rate_hz() const { rtc::CritScope lock(&crit_sect_); - return last_output_sample_rate_hz_; + return delayed_last_output_sample_rate_hz_.value_or( + last_output_sample_rate_hz_); } absl::optional<NetEq::DecoderFormat> NetEqImpl::GetDecoderFormat( @@ -1988,8 +2046,9 @@ int NetEqImpl::ExtractPackets(size_t required_samples, extracted_samples = packet->timestamp - first_timestamp + packet_duration; RTC_DCHECK(controller_); - stats_->JitterBufferDelay(packet_duration, waiting_time_ms, - controller_->TargetLevelMs()); + stats_->JitterBufferDelay( + packet_duration, waiting_time_ms + output_delay_chain_ms_, + controller_->TargetLevelMs() + output_delay_chain_ms_); packet_list->push_back(std::move(*packet)); // Store packet in list. packet = absl::nullopt; // Ensure it's never used after the move. diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.h index 956cb6ef178..623968aefd1 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_impl.h @@ -402,6 +402,22 @@ class NetEqImpl : public webrtc::NetEq { bool no_time_stretching_ RTC_GUARDED_BY(crit_sect_); // Only used for test. rtc::BufferT<int16_t> concealment_audio_ RTC_GUARDED_BY(crit_sect_); const bool enable_rtx_handling_ RTC_GUARDED_BY(crit_sect_); + // Data members used for adding extra delay to the output of NetEq. + // The delay in ms (which is 10 times the number of elements in + // output_delay_chain_). + const int output_delay_chain_ms_ RTC_GUARDED_BY(crit_sect_); + // Vector of AudioFrames which contains the delayed audio. Accessed as a + // circular buffer. + std::vector<AudioFrame> output_delay_chain_ RTC_GUARDED_BY(crit_sect_); + // Index into output_delay_chain_. + size_t output_delay_chain_ix_ RTC_GUARDED_BY(crit_sect_) = 0; + // Did output_delay_chain_ get populated yet? + bool output_delay_chain_empty_ RTC_GUARDED_BY(crit_sect_) = true; + // Contains the sample rate of the AudioFrame last emitted from the delay + // chain. If the extra output delay chain is not used, or if no audio has been + // emitted yet, the variable is empty. + absl::optional<int> delayed_last_output_sample_rate_hz_ + RTC_GUARDED_BY(crit_sect_); private: RTC_DISALLOW_COPY_AND_ASSIGN(NetEqImpl); diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc index d35c44c4c26..df346227226 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_network_stats_unittest.cc @@ -52,10 +52,10 @@ class MockAudioDecoder final : public AudioDecoder { : sample_rate_hz_(sample_rate_hz), num_channels_(num_channels), fec_enabled_(false) {} - ~MockAudioDecoder() /* override */ { Die(); } - MOCK_METHOD0(Die, void()); + ~MockAudioDecoder() override { Die(); } + MOCK_METHOD(void, Die, ()); - MOCK_METHOD0(Reset, void()); + MOCK_METHOD(void, Reset, (), (override)); class MockFrame : public AudioDecoder::EncodedAudioFrame { public: diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_unittest.cc index d78e2c64886..f5fb6479658 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/neteq_unittest.cc @@ -1102,5 +1102,186 @@ TEST(NetEqNoTimeStretchingMode, RunTest) { EXPECT_EQ(0, stats.preemptive_rate); } +namespace { +// Helper classes and data types and functions for NetEqOutputDelayTest. + +class VectorAudioSink : public AudioSink { + public: + // Does not take ownership of the vector. + VectorAudioSink(std::vector<int16_t>* output_vector) : v_(output_vector) {} + + virtual ~VectorAudioSink() = default; + + bool WriteArray(const int16_t* audio, size_t num_samples) override { + v_->reserve(v_->size() + num_samples); + for (size_t i = 0; i < num_samples; ++i) { + v_->push_back(audio[i]); + } + return true; + } + + private: + std::vector<int16_t>* const v_; +}; + +struct TestResult { + NetEqLifetimeStatistics lifetime_stats; + NetEqNetworkStatistics network_stats; + absl::optional<uint32_t> playout_timestamp; + int target_delay_ms; + int filtered_current_delay_ms; + int sample_rate_hz; +}; + +// This class is used as callback object to NetEqTest to collect some stats +// at the end of the simulation. +class SimEndStatsCollector : public NetEqSimulationEndedCallback { + public: + SimEndStatsCollector(TestResult& result) : result_(result) {} + + void SimulationEnded(int64_t /*simulation_time_ms*/, NetEq* neteq) override { + result_.playout_timestamp = neteq->GetPlayoutTimestamp(); + result_.target_delay_ms = neteq->TargetDelayMs(); + result_.filtered_current_delay_ms = neteq->FilteredCurrentDelayMs(); + result_.sample_rate_hz = neteq->last_output_sample_rate_hz(); + } + + private: + TestResult& result_; +}; + +TestResult DelayLineNetEqTest(int delay_ms, + std::vector<int16_t>* output_vector) { + NetEq::Config config; + config.for_test_no_time_stretching = true; + config.extra_output_delay_ms = delay_ms; + auto codecs = NetEqTest::StandardDecoderMap(); + NetEqPacketSourceInput::RtpHeaderExtensionMap rtp_ext_map = { + {1, kRtpExtensionAudioLevel}, + {3, kRtpExtensionAbsoluteSendTime}, + {5, kRtpExtensionTransportSequenceNumber}, + {7, kRtpExtensionVideoContentType}, + {8, kRtpExtensionVideoTiming}}; + std::unique_ptr<NetEqInput> input = std::make_unique<NetEqRtpDumpInput>( + webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"), + rtp_ext_map, absl::nullopt /*No SSRC filter*/); + std::unique_ptr<TimeLimitedNetEqInput> input_time_limit( + new TimeLimitedNetEqInput(std::move(input), 10000)); + std::unique_ptr<AudioSink> output = + std::make_unique<VectorAudioSink>(output_vector); + + TestResult result; + SimEndStatsCollector stats_collector(result); + NetEqTest::Callbacks callbacks; + callbacks.simulation_ended_callback = &stats_collector; + + NetEqTest test(config, CreateBuiltinAudioDecoderFactory(), codecs, + /*text_log=*/nullptr, /*neteq_factory=*/nullptr, + /*input=*/std::move(input_time_limit), std::move(output), + callbacks); + test.Run(); + result.lifetime_stats = test.LifetimeStats(); + result.network_stats = test.SimulationStats(); + return result; +} +} // namespace + +// Tests the extra output delay functionality of NetEq. +TEST(NetEqOutputDelayTest, RunTest) { + std::vector<int16_t> output; + const auto result_no_delay = DelayLineNetEqTest(0, &output); + std::vector<int16_t> output_delayed; + constexpr int kDelayMs = 100; + const auto result_delay = DelayLineNetEqTest(kDelayMs, &output_delayed); + + // Verify that the loss concealment remains unchanged. The point of the delay + // is to not affect the jitter buffering behavior. + // First verify that there are concealments in the test. + EXPECT_GT(result_no_delay.lifetime_stats.concealed_samples, 0u); + // And that not all of the output is concealment. + EXPECT_GT(result_no_delay.lifetime_stats.total_samples_received, + result_no_delay.lifetime_stats.concealed_samples); + // Now verify that they remain unchanged by the delay. + EXPECT_EQ(result_no_delay.lifetime_stats.concealed_samples, + result_delay.lifetime_stats.concealed_samples); + // Accelerate and pre-emptive expand should also be unchanged. + EXPECT_EQ(result_no_delay.lifetime_stats.inserted_samples_for_deceleration, + result_delay.lifetime_stats.inserted_samples_for_deceleration); + EXPECT_EQ(result_no_delay.lifetime_stats.removed_samples_for_acceleration, + result_delay.lifetime_stats.removed_samples_for_acceleration); + // Verify that delay stats are increased with the delay chain. + EXPECT_EQ( + result_no_delay.lifetime_stats.jitter_buffer_delay_ms + + kDelayMs * result_no_delay.lifetime_stats.jitter_buffer_emitted_count, + result_delay.lifetime_stats.jitter_buffer_delay_ms); + EXPECT_EQ( + result_no_delay.lifetime_stats.jitter_buffer_target_delay_ms + + kDelayMs * result_no_delay.lifetime_stats.jitter_buffer_emitted_count, + result_delay.lifetime_stats.jitter_buffer_target_delay_ms); + EXPECT_EQ(result_no_delay.network_stats.current_buffer_size_ms + kDelayMs, + result_delay.network_stats.current_buffer_size_ms); + EXPECT_EQ(result_no_delay.network_stats.preferred_buffer_size_ms + kDelayMs, + result_delay.network_stats.preferred_buffer_size_ms); + EXPECT_EQ(result_no_delay.network_stats.mean_waiting_time_ms + kDelayMs, + result_delay.network_stats.mean_waiting_time_ms); + EXPECT_EQ(result_no_delay.network_stats.median_waiting_time_ms + kDelayMs, + result_delay.network_stats.median_waiting_time_ms); + EXPECT_EQ(result_no_delay.network_stats.min_waiting_time_ms + kDelayMs, + result_delay.network_stats.min_waiting_time_ms); + EXPECT_EQ(result_no_delay.network_stats.max_waiting_time_ms + kDelayMs, + result_delay.network_stats.max_waiting_time_ms); + + ASSERT_TRUE(result_no_delay.playout_timestamp); + ASSERT_TRUE(result_delay.playout_timestamp); + EXPECT_EQ(*result_no_delay.playout_timestamp - + static_cast<uint32_t>( + kDelayMs * + rtc::CheckedDivExact(result_no_delay.sample_rate_hz, 1000)), + *result_delay.playout_timestamp); + EXPECT_EQ(result_no_delay.target_delay_ms + kDelayMs, + result_delay.target_delay_ms); + EXPECT_EQ(result_no_delay.filtered_current_delay_ms + kDelayMs, + result_delay.filtered_current_delay_ms); + + // Verify expected delay in decoded signal. The test vector uses 8 kHz sample + // rate, so the delay will be 8 times the delay in ms. + constexpr size_t kExpectedDelaySamples = kDelayMs * 8; + for (size_t i = 0; + i < output.size() && i + kExpectedDelaySamples < output_delayed.size(); + ++i) { + EXPECT_EQ(output[i], output_delayed[i + kExpectedDelaySamples]); + } +} + +// Tests the extra output delay functionality of NetEq when configured via +// field trial. +TEST(NetEqOutputDelayTest, RunTestWithFieldTrial) { + test::ScopedFieldTrials field_trial( + "WebRTC-Audio-NetEqExtraDelay/Enabled-50/"); + constexpr int kExpectedDelayMs = 50; + std::vector<int16_t> output; + const auto result = DelayLineNetEqTest(0, &output); + + // The base delay values are taken from the resuts of the non-delayed case in + // NetEqOutputDelayTest.RunTest above. + EXPECT_EQ(10 + kExpectedDelayMs, result.target_delay_ms); + EXPECT_EQ(24 + kExpectedDelayMs, result.filtered_current_delay_ms); +} + +// Set a non-multiple-of-10 value in the field trial, and verify that we don't +// crash, and that the result is rounded down. +TEST(NetEqOutputDelayTest, RunTestWithFieldTrialOddValue) { + test::ScopedFieldTrials field_trial( + "WebRTC-Audio-NetEqExtraDelay/Enabled-103/"); + constexpr int kRoundedDelayMs = 100; + std::vector<int16_t> output; + const auto result = DelayLineNetEqTest(0, &output); + + // The base delay values are taken from the resuts of the non-delayed case in + // NetEqOutputDelayTest.RunTest above. + EXPECT_EQ(10 + kRoundedDelayMs, result.target_delay_ms); + EXPECT_EQ(24 + kRoundedDelayMs, result.filtered_current_delay_ms); +} + } // namespace test } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc index 0aff955fd75..40e7d5371af 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/packet_buffer_unittest.cc @@ -31,13 +31,14 @@ using ::testing::StrictMock; namespace { class MockEncodedAudioFrame : public webrtc::AudioDecoder::EncodedAudioFrame { public: - MOCK_CONST_METHOD0(Duration, size_t()); + MOCK_METHOD(size_t, Duration, (), (const, override)); - MOCK_CONST_METHOD0(IsDtxPacket, bool()); + MOCK_METHOD(bool, IsDtxPacket, (), (const, override)); - MOCK_CONST_METHOD1( - Decode, - absl::optional<DecodeResult>(rtc::ArrayView<int16_t> decoded)); + MOCK_METHOD(absl::optional<DecodeResult>, + Decode, + (rtc::ArrayView<int16_t> decoded), + (const, override)); }; // Helper class to generate packets. Packets must be deleted by the user. diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc index 3f06b1cfc44..337f54ed6ed 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.cc @@ -33,7 +33,8 @@ NetEqStatsPlotter::NetEqStatsPlotter(bool make_matlab_plot, stats_getter_.reset(new NetEqStatsGetter(std::move(delay_analyzer))); } -void NetEqStatsPlotter::SimulationEnded(int64_t simulation_time_ms) { +void NetEqStatsPlotter::SimulationEnded(int64_t simulation_time_ms, + NetEq* /*neteq*/) { if (make_matlab_plot_) { auto matlab_script_name = base_file_name_; std::replace(matlab_script_name.begin(), matlab_script_name.end(), '.', diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h index c4df24e0731..d6918670fd8 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_stats_plotter.h @@ -28,7 +28,7 @@ class NetEqStatsPlotter : public NetEqSimulationEndedCallback { bool show_concealment_events, std::string base_file_name); - void SimulationEnded(int64_t simulation_time_ms) override; + void SimulationEnded(int64_t simulation_time_ms, NetEq* neteq) override; NetEqStatsGetter* stats_getter() { return stats_getter_.get(); } diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc index f8b6161a98c..a263a737217 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.cc @@ -91,7 +91,8 @@ int64_t NetEqTest::Run() { simulation_time += step_result.simulation_step_ms; } while (!step_result.is_simulation_finished); if (callbacks_.simulation_ended_callback) { - callbacks_.simulation_ended_callback->SimulationEnded(simulation_time); + callbacks_.simulation_ended_callback->SimulationEnded(simulation_time, + neteq_.get()); } return simulation_time; } diff --git a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.h b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.h index 0a6c24f3d66..3b787a6cfbd 100644 --- a/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.h +++ b/chromium/third_party/webrtc/modules/audio_coding/neteq/tools/neteq_test.h @@ -61,7 +61,7 @@ class NetEqGetAudioCallback { class NetEqSimulationEndedCallback { public: virtual ~NetEqSimulationEndedCallback() = default; - virtual void SimulationEnded(int64_t simulation_time_ms) = 0; + virtual void SimulationEnded(int64_t simulation_time_ms, NetEq* neteq) = 0; }; // Class that provides an input--output test for NetEq. The input (both packets diff --git a/chromium/third_party/webrtc/modules/audio_device/BUILD.gn b/chromium/third_party/webrtc/modules/audio_device/BUILD.gn index 2ce0ae20e88..0d1ee81b47a 100644 --- a/chromium/third_party/webrtc/modules/audio_device/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_device/BUILD.gn @@ -144,8 +144,8 @@ rtc_source_set("audio_device_module_from_input_and_output") { "../../api/task_queue", "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } } @@ -385,8 +385,8 @@ if (rtc_include_tests) { "../../test:fileutils", "../../test:test_support", "../utility", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] if (is_linux || is_mac || is_win) { sources += [ "audio_device_unittest.cc" ] } diff --git a/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.cc b/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.cc index 776f0cfd704..daaeeca1eaf 100644 --- a/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.cc +++ b/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.cc @@ -19,6 +19,7 @@ #include "rtc_base/logging.h" #include "rtc_base/platform_thread.h" #include "system_wrappers/include/field_trial.h" +#include "system_wrappers/include/metrics.h" namespace webrtc { @@ -27,13 +28,15 @@ AudioTrackJni::JavaAudioTrack::JavaAudioTrack( NativeRegistration* native_reg, std::unique_ptr<GlobalRef> audio_track) : audio_track_(std::move(audio_track)), - init_playout_(native_reg->GetMethodId("initPlayout", "(IID)Z")), + init_playout_(native_reg->GetMethodId("initPlayout", "(IID)I")), start_playout_(native_reg->GetMethodId("startPlayout", "()Z")), stop_playout_(native_reg->GetMethodId("stopPlayout", "()Z")), set_stream_volume_(native_reg->GetMethodId("setStreamVolume", "(I)Z")), get_stream_max_volume_( native_reg->GetMethodId("getStreamMaxVolume", "()I")), - get_stream_volume_(native_reg->GetMethodId("getStreamVolume", "()I")) {} + get_stream_volume_(native_reg->GetMethodId("getStreamVolume", "()I")), + get_buffer_size_in_frames_( + native_reg->GetMethodId("getBufferSizeInFrames", "()I")) {} AudioTrackJni::JavaAudioTrack::~JavaAudioTrack() {} @@ -45,8 +48,29 @@ bool AudioTrackJni::JavaAudioTrack::InitPlayout(int sample_rate, int channels) { nullptr); if (buffer_size_factor == 0) buffer_size_factor = 1.0; - return audio_track_->CallBooleanMethod(init_playout_, sample_rate, channels, - buffer_size_factor); + int requested_buffer_size_bytes = audio_track_->CallIntMethod( + init_playout_, sample_rate, channels, buffer_size_factor); + // Update UMA histograms for both the requested and actual buffer size. + if (requested_buffer_size_bytes >= 0) { + // To avoid division by zero, we assume the sample rate is 48k if an invalid + // value is found. + sample_rate = sample_rate <= 0 ? 48000 : sample_rate; + // This calculation assumes that audio is mono. + const int requested_buffer_size_ms = + (requested_buffer_size_bytes * 1000) / (2 * sample_rate); + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AndroidNativeRequestedAudioBufferSizeMs", + requested_buffer_size_ms, 0, 1000, 100); + int actual_buffer_size_frames = + audio_track_->CallIntMethod(get_buffer_size_in_frames_); + if (actual_buffer_size_frames >= 0) { + const int actual_buffer_size_ms = + actual_buffer_size_frames * 1000 / sample_rate; + RTC_HISTOGRAM_COUNTS("WebRTC.Audio.AndroidNativeAudioBufferSizeMs", + actual_buffer_size_ms, 0, 1000, 100); + } + return true; + } + return false; } bool AudioTrackJni::JavaAudioTrack::StartPlayout() { diff --git a/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.h b/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.h index 6303d754c8a..529a9013e80 100644 --- a/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.h +++ b/chromium/third_party/webrtc/modules/audio_device/android/audio_track_jni.h @@ -62,6 +62,7 @@ class AudioTrackJni { jmethodID set_stream_volume_; jmethodID get_stream_max_volume_; jmethodID get_stream_volume_; + jmethodID get_buffer_size_in_frames_; }; explicit AudioTrackJni(AudioManager* audio_manager); diff --git a/chromium/third_party/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java b/chromium/third_party/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java index 3023c99fa26..7e6ad5acf4f 100644 --- a/chromium/third_party/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java +++ b/chromium/third_party/webrtc/modules/audio_device/android/java/src/org/webrtc/voiceengine/WebRtcAudioTrack.java @@ -215,7 +215,7 @@ public class WebRtcAudioTrack { } } - private boolean initPlayout(int sampleRate, int channels, double bufferSizeFactor) { + private int initPlayout(int sampleRate, int channels, double bufferSizeFactor) { threadChecker.checkIsOnValidThread(); Logging.d(TAG, "initPlayout(sampleRate=" + sampleRate + ", channels=" + channels @@ -244,14 +244,14 @@ public class WebRtcAudioTrack { // can happen that |minBufferSizeInBytes| contains an invalid value. if (minBufferSizeInBytes < byteBuffer.capacity()) { reportWebRtcAudioTrackInitError("AudioTrack.getMinBufferSize returns an invalid value."); - return false; + return -1; } // Ensure that prevision audio session was stopped correctly before trying // to create a new AudioTrack. if (audioTrack != null) { reportWebRtcAudioTrackInitError("Conflict with existing AudioTrack."); - return false; + return -1; } try { // Create an AudioTrack object and initialize its associated audio buffer. @@ -273,7 +273,7 @@ public class WebRtcAudioTrack { } catch (IllegalArgumentException e) { reportWebRtcAudioTrackInitError(e.getMessage()); releaseAudioResources(); - return false; + return -1; } // It can happen that an AudioTrack is created but it was not successfully @@ -282,11 +282,11 @@ public class WebRtcAudioTrack { if (audioTrack == null || audioTrack.getState() != AudioTrack.STATE_INITIALIZED) { reportWebRtcAudioTrackInitError("Initialization of audio track failed."); releaseAudioResources(); - return false; + return -1; } logMainParameters(); logMainParametersExtended(); - return true; + return minBufferSizeInBytes; } private boolean startPlayout() { @@ -433,6 +433,13 @@ public class WebRtcAudioTrack { } } + private int getBufferSizeInFrames() { + if (Build.VERSION.SDK_INT >= 23) { + return audioTrack.getBufferSizeInFrames(); + } + return -1; + } + private void logBufferCapacityInFrames() { if (Build.VERSION.SDK_INT >= 24) { Logging.d(TAG, diff --git a/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_device.h b/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_device.h index a05e64e6c9d..0ca19de156a 100644 --- a/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_device.h +++ b/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_device.h @@ -32,76 +32,123 @@ class MockAudioDeviceModule : public AudioDeviceModule { } // AudioDeviceModule. - MOCK_CONST_METHOD1(ActiveAudioLayer, int32_t(AudioLayer* audioLayer)); - MOCK_METHOD1(RegisterAudioCallback, int32_t(AudioTransport* audioCallback)); - MOCK_METHOD0(Init, int32_t()); - MOCK_METHOD0(Terminate, int32_t()); - MOCK_CONST_METHOD0(Initialized, bool()); - MOCK_METHOD0(PlayoutDevices, int16_t()); - MOCK_METHOD0(RecordingDevices, int16_t()); - MOCK_METHOD3(PlayoutDeviceName, - int32_t(uint16_t index, - char name[kAdmMaxDeviceNameSize], - char guid[kAdmMaxGuidSize])); - MOCK_METHOD3(RecordingDeviceName, - int32_t(uint16_t index, - char name[kAdmMaxDeviceNameSize], - char guid[kAdmMaxGuidSize])); - MOCK_METHOD1(SetPlayoutDevice, int32_t(uint16_t index)); - MOCK_METHOD1(SetPlayoutDevice, int32_t(WindowsDeviceType device)); - MOCK_METHOD1(SetRecordingDevice, int32_t(uint16_t index)); - MOCK_METHOD1(SetRecordingDevice, int32_t(WindowsDeviceType device)); - MOCK_METHOD1(PlayoutIsAvailable, int32_t(bool* available)); - MOCK_METHOD0(InitPlayout, int32_t()); - MOCK_CONST_METHOD0(PlayoutIsInitialized, bool()); - MOCK_METHOD1(RecordingIsAvailable, int32_t(bool* available)); - MOCK_METHOD0(InitRecording, int32_t()); - MOCK_CONST_METHOD0(RecordingIsInitialized, bool()); - MOCK_METHOD0(StartPlayout, int32_t()); - MOCK_METHOD0(StopPlayout, int32_t()); - MOCK_CONST_METHOD0(Playing, bool()); - MOCK_METHOD0(StartRecording, int32_t()); - MOCK_METHOD0(StopRecording, int32_t()); - MOCK_CONST_METHOD0(Recording, bool()); - MOCK_METHOD1(SetAGC, int32_t(bool enable)); - MOCK_CONST_METHOD0(AGC, bool()); - MOCK_METHOD0(InitSpeaker, int32_t()); - MOCK_CONST_METHOD0(SpeakerIsInitialized, bool()); - MOCK_METHOD0(InitMicrophone, int32_t()); - MOCK_CONST_METHOD0(MicrophoneIsInitialized, bool()); - MOCK_METHOD1(SpeakerVolumeIsAvailable, int32_t(bool* available)); - MOCK_METHOD1(SetSpeakerVolume, int32_t(uint32_t volume)); - MOCK_CONST_METHOD1(SpeakerVolume, int32_t(uint32_t* volume)); - MOCK_CONST_METHOD1(MaxSpeakerVolume, int32_t(uint32_t* maxVolume)); - MOCK_CONST_METHOD1(MinSpeakerVolume, int32_t(uint32_t* minVolume)); - MOCK_METHOD1(MicrophoneVolumeIsAvailable, int32_t(bool* available)); - MOCK_METHOD1(SetMicrophoneVolume, int32_t(uint32_t volume)); - MOCK_CONST_METHOD1(MicrophoneVolume, int32_t(uint32_t* volume)); - MOCK_CONST_METHOD1(MaxMicrophoneVolume, int32_t(uint32_t* maxVolume)); - MOCK_CONST_METHOD1(MinMicrophoneVolume, int32_t(uint32_t* minVolume)); - MOCK_METHOD1(SpeakerMuteIsAvailable, int32_t(bool* available)); - MOCK_METHOD1(SetSpeakerMute, int32_t(bool enable)); - MOCK_CONST_METHOD1(SpeakerMute, int32_t(bool* enabled)); - MOCK_METHOD1(MicrophoneMuteIsAvailable, int32_t(bool* available)); - MOCK_METHOD1(SetMicrophoneMute, int32_t(bool enable)); - MOCK_CONST_METHOD1(MicrophoneMute, int32_t(bool* enabled)); - MOCK_CONST_METHOD1(StereoPlayoutIsAvailable, int32_t(bool* available)); - MOCK_METHOD1(SetStereoPlayout, int32_t(bool enable)); - MOCK_CONST_METHOD1(StereoPlayout, int32_t(bool* enabled)); - MOCK_CONST_METHOD1(StereoRecordingIsAvailable, int32_t(bool* available)); - MOCK_METHOD1(SetStereoRecording, int32_t(bool enable)); - MOCK_CONST_METHOD1(StereoRecording, int32_t(bool* enabled)); - MOCK_CONST_METHOD1(PlayoutDelay, int32_t(uint16_t* delayMS)); - MOCK_CONST_METHOD0(BuiltInAECIsAvailable, bool()); - MOCK_CONST_METHOD0(BuiltInAGCIsAvailable, bool()); - MOCK_CONST_METHOD0(BuiltInNSIsAvailable, bool()); - MOCK_METHOD1(EnableBuiltInAEC, int32_t(bool enable)); - MOCK_METHOD1(EnableBuiltInAGC, int32_t(bool enable)); - MOCK_METHOD1(EnableBuiltInNS, int32_t(bool enable)); - MOCK_CONST_METHOD0(GetPlayoutUnderrunCount, int32_t()); + MOCK_METHOD(int32_t, + ActiveAudioLayer, + (AudioLayer * audioLayer), + (const, override)); + MOCK_METHOD(int32_t, + RegisterAudioCallback, + (AudioTransport * audioCallback), + (override)); + MOCK_METHOD(int32_t, Init, (), (override)); + MOCK_METHOD(int32_t, Terminate, (), (override)); + MOCK_METHOD(bool, Initialized, (), (const, override)); + MOCK_METHOD(int16_t, PlayoutDevices, (), (override)); + MOCK_METHOD(int16_t, RecordingDevices, (), (override)); + MOCK_METHOD(int32_t, + PlayoutDeviceName, + (uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]), + (override)); + MOCK_METHOD(int32_t, + RecordingDeviceName, + (uint16_t index, + char name[kAdmMaxDeviceNameSize], + char guid[kAdmMaxGuidSize]), + (override)); + MOCK_METHOD(int32_t, SetPlayoutDevice, (uint16_t index), (override)); + MOCK_METHOD(int32_t, + SetPlayoutDevice, + (WindowsDeviceType device), + (override)); + MOCK_METHOD(int32_t, SetRecordingDevice, (uint16_t index), (override)); + MOCK_METHOD(int32_t, + SetRecordingDevice, + (WindowsDeviceType device), + (override)); + MOCK_METHOD(int32_t, PlayoutIsAvailable, (bool* available), (override)); + MOCK_METHOD(int32_t, InitPlayout, (), (override)); + MOCK_METHOD(bool, PlayoutIsInitialized, (), (const, override)); + MOCK_METHOD(int32_t, RecordingIsAvailable, (bool* available), (override)); + MOCK_METHOD(int32_t, InitRecording, (), (override)); + MOCK_METHOD(bool, RecordingIsInitialized, (), (const, override)); + MOCK_METHOD(int32_t, StartPlayout, (), (override)); + MOCK_METHOD(int32_t, StopPlayout, (), (override)); + MOCK_METHOD(bool, Playing, (), (const, override)); + MOCK_METHOD(int32_t, StartRecording, (), (override)); + MOCK_METHOD(int32_t, StopRecording, (), (override)); + MOCK_METHOD(bool, Recording, (), (const, override)); + MOCK_METHOD(int32_t, InitSpeaker, (), (override)); + MOCK_METHOD(bool, SpeakerIsInitialized, (), (const, override)); + MOCK_METHOD(int32_t, InitMicrophone, (), (override)); + MOCK_METHOD(bool, MicrophoneIsInitialized, (), (const, override)); + MOCK_METHOD(int32_t, SpeakerVolumeIsAvailable, (bool* available), (override)); + MOCK_METHOD(int32_t, SetSpeakerVolume, (uint32_t volume), (override)); + MOCK_METHOD(int32_t, SpeakerVolume, (uint32_t * volume), (const, override)); + MOCK_METHOD(int32_t, + MaxSpeakerVolume, + (uint32_t * maxVolume), + (const, override)); + MOCK_METHOD(int32_t, + MinSpeakerVolume, + (uint32_t * minVolume), + (const, override)); + MOCK_METHOD(int32_t, + MicrophoneVolumeIsAvailable, + (bool* available), + (override)); + MOCK_METHOD(int32_t, SetMicrophoneVolume, (uint32_t volume), (override)); + MOCK_METHOD(int32_t, + MicrophoneVolume, + (uint32_t * volume), + (const, override)); + MOCK_METHOD(int32_t, + MaxMicrophoneVolume, + (uint32_t * maxVolume), + (const, override)); + MOCK_METHOD(int32_t, + MinMicrophoneVolume, + (uint32_t * minVolume), + (const, override)); + MOCK_METHOD(int32_t, SpeakerMuteIsAvailable, (bool* available), (override)); + MOCK_METHOD(int32_t, SetSpeakerMute, (bool enable), (override)); + MOCK_METHOD(int32_t, SpeakerMute, (bool* enabled), (const, override)); + MOCK_METHOD(int32_t, + MicrophoneMuteIsAvailable, + (bool* available), + (override)); + MOCK_METHOD(int32_t, SetMicrophoneMute, (bool enable), (override)); + MOCK_METHOD(int32_t, MicrophoneMute, (bool* enabled), (const, override)); + MOCK_METHOD(int32_t, + StereoPlayoutIsAvailable, + (bool* available), + (const, override)); + MOCK_METHOD(int32_t, SetStereoPlayout, (bool enable), (override)); + MOCK_METHOD(int32_t, StereoPlayout, (bool* enabled), (const, override)); + MOCK_METHOD(int32_t, + StereoRecordingIsAvailable, + (bool* available), + (const, override)); + MOCK_METHOD(int32_t, SetStereoRecording, (bool enable), (override)); + MOCK_METHOD(int32_t, StereoRecording, (bool* enabled), (const, override)); + MOCK_METHOD(int32_t, PlayoutDelay, (uint16_t * delayMS), (const, override)); + MOCK_METHOD(bool, BuiltInAECIsAvailable, (), (const, override)); + MOCK_METHOD(bool, BuiltInAGCIsAvailable, (), (const, override)); + MOCK_METHOD(bool, BuiltInNSIsAvailable, (), (const, override)); + MOCK_METHOD(int32_t, EnableBuiltInAEC, (bool enable), (override)); + MOCK_METHOD(int32_t, EnableBuiltInAGC, (bool enable), (override)); + MOCK_METHOD(int32_t, EnableBuiltInNS, (bool enable), (override)); + MOCK_METHOD(int32_t, GetPlayoutUnderrunCount, (), (const, override)); #if defined(WEBRTC_IOS) - MOCK_CONST_METHOD1(GetPlayoutAudioParameters, int(AudioParameters* params)); - MOCK_CONST_METHOD1(GetRecordAudioParameters, int(AudioParameters* params)); + MOCK_METHOD(int, + GetPlayoutAudioParameters, + (AudioParameters * params), + (const, override)); + MOCK_METHOD(int, + GetRecordAudioParameters, + (AudioParameters * params), + (const, override)); #endif // WEBRTC_IOS }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_transport.h b/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_transport.h index ebdfbc70154..8f71a2d71f7 100644 --- a/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_transport.h +++ b/chromium/third_party/webrtc/modules/audio_device/include/mock_audio_transport.h @@ -22,36 +22,42 @@ class MockAudioTransport : public AudioTransport { MockAudioTransport() {} ~MockAudioTransport() {} - MOCK_METHOD10(RecordedDataIsAvailable, - int32_t(const void* audioSamples, - const size_t nSamples, - const size_t nBytesPerSample, - const size_t nChannels, - const uint32_t samplesPerSec, - const uint32_t totalDelayMS, - const int32_t clockDrift, - const uint32_t currentMicLevel, - const bool keyPressed, - uint32_t& newMicLevel)); - - MOCK_METHOD8(NeedMorePlayData, - int32_t(const size_t nSamples, - const size_t nBytesPerSample, - const size_t nChannels, - const uint32_t samplesPerSec, - void* audioSamples, - size_t& nSamplesOut, - int64_t* elapsed_time_ms, - int64_t* ntp_time_ms)); - - MOCK_METHOD7(PullRenderData, - void(int bits_per_sample, - int sample_rate, - size_t number_of_channels, - size_t number_of_frames, - void* audio_data, - int64_t* elapsed_time_ms, - int64_t* ntp_time_ms)); + MOCK_METHOD(int32_t, + RecordedDataIsAvailable, + (const void* audioSamples, + const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + const uint32_t totalDelayMS, + const int32_t clockDrift, + const uint32_t currentMicLevel, + const bool keyPressed, + uint32_t& newMicLevel), + (override)); + + MOCK_METHOD(int32_t, + NeedMorePlayData, + (const size_t nSamples, + const size_t nBytesPerSample, + const size_t nChannels, + const uint32_t samplesPerSec, + void* audioSamples, + size_t& nSamplesOut, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms), + (override)); + + MOCK_METHOD(void, + PullRenderData, + (int bits_per_sample, + int sample_rate, + size_t number_of_channels, + size_t number_of_frames, + void* audio_data, + int64_t* elapsed_time_ms, + int64_t* ntp_time_ms), + (override)); }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.cc b/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.cc index e894cf309a4..9368c312639 100644 --- a/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.cc +++ b/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.cc @@ -410,7 +410,10 @@ int32_t AudioDeviceMac::SpeakerIsAvailable(bool& available) { int32_t AudioDeviceMac::InitSpeaker() { rtc::CritScope lock(&_critSect); + return InitSpeakerLocked(); +} +int32_t AudioDeviceMac::InitSpeakerLocked() { if (_playing) { return -1; } @@ -458,7 +461,10 @@ int32_t AudioDeviceMac::MicrophoneIsAvailable(bool& available) { int32_t AudioDeviceMac::InitMicrophone() { rtc::CritScope lock(&_critSect); + return InitMicrophoneLocked(); +} +int32_t AudioDeviceMac::InitMicrophoneLocked() { if (_recording) { return -1; } @@ -960,7 +966,7 @@ int32_t AudioDeviceMac::InitPlayout() { } // Initialize the speaker (devices might have been added or removed) - if (InitSpeaker() == -1) { + if (InitSpeakerLocked() == -1) { RTC_LOG(LS_WARNING) << "InitSpeaker() failed"; } @@ -1098,7 +1104,7 @@ int32_t AudioDeviceMac::InitRecording() { } // Initialize the microphone (devices might have been added or removed) - if (InitMicrophone() == -1) { + if (InitMicrophoneLocked() == -1) { RTC_LOG(LS_WARNING) << "InitMicrophone() failed"; } diff --git a/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.h b/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.h index d7076a3c1cc..8d0e7fa571b 100644 --- a/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.h +++ b/chromium/third_party/webrtc/modules/audio_device/mac/audio_device_mac.h @@ -69,8 +69,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { AudioDeviceModule::AudioLayer& audioLayer) const; // Main initializaton and termination - virtual InitStatus Init(); - virtual int32_t Terminate(); + virtual InitStatus Init() RTC_LOCKS_EXCLUDED(_critSect); + virtual int32_t Terminate() RTC_LOCKS_EXCLUDED(_critSect); virtual bool Initialized() const; // Device enumeration @@ -84,7 +84,8 @@ class AudioDeviceMac : public AudioDeviceGeneric { char guid[kAdmMaxGuidSize]); // Device selection - virtual int32_t SetPlayoutDevice(uint16_t index); + virtual int32_t SetPlayoutDevice(uint16_t index) + RTC_LOCKS_EXCLUDED(_critSect); virtual int32_t SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType device); virtual int32_t SetRecordingDevice(uint16_t index); virtual int32_t SetRecordingDevice( @@ -92,24 +93,24 @@ class AudioDeviceMac : public AudioDeviceGeneric { // Audio transport initialization virtual int32_t PlayoutIsAvailable(bool& available); - virtual int32_t InitPlayout(); + virtual int32_t InitPlayout() RTC_LOCKS_EXCLUDED(_critSect); virtual bool PlayoutIsInitialized() const; virtual int32_t RecordingIsAvailable(bool& available); - virtual int32_t InitRecording(); + virtual int32_t InitRecording() RTC_LOCKS_EXCLUDED(_critSect); virtual bool RecordingIsInitialized() const; // Audio transport control - virtual int32_t StartPlayout(); - virtual int32_t StopPlayout(); + virtual int32_t StartPlayout() RTC_LOCKS_EXCLUDED(_critSect); + virtual int32_t StopPlayout() RTC_LOCKS_EXCLUDED(_critSect); virtual bool Playing() const; - virtual int32_t StartRecording(); - virtual int32_t StopRecording(); + virtual int32_t StartRecording() RTC_LOCKS_EXCLUDED(_critSect); + virtual int32_t StopRecording() RTC_LOCKS_EXCLUDED(_critSect); virtual bool Recording() const; // Audio mixer initialization - virtual int32_t InitSpeaker(); + virtual int32_t InitSpeaker() RTC_LOCKS_EXCLUDED(_critSect); virtual bool SpeakerIsInitialized() const; - virtual int32_t InitMicrophone(); + virtual int32_t InitMicrophone() RTC_LOCKS_EXCLUDED(_critSect); virtual bool MicrophoneIsInitialized() const; // Speaker volume controls @@ -147,9 +148,13 @@ class AudioDeviceMac : public AudioDeviceGeneric { // Delay information and control virtual int32_t PlayoutDelay(uint16_t& delayMS) const; - virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer); + virtual void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) + RTC_LOCKS_EXCLUDED(_critSect); private: + int32_t InitSpeakerLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(_critSect); + int32_t InitMicrophoneLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(_critSect); + virtual int32_t MicrophoneIsAvailable(bool& available); virtual int32_t SpeakerIsAvailable(bool& available); @@ -229,13 +234,15 @@ class AudioDeviceMac : public AudioDeviceGeneric { OSStatus implDeviceIOProc(const AudioBufferList* inputData, const AudioTimeStamp* inputTime, AudioBufferList* outputData, - const AudioTimeStamp* outputTime); + const AudioTimeStamp* outputTime) + RTC_LOCKS_EXCLUDED(_critSect); OSStatus implOutConverterProc(UInt32* numberDataPackets, AudioBufferList* data); OSStatus implInDeviceIOProc(const AudioBufferList* inputData, - const AudioTimeStamp* inputTime); + const AudioTimeStamp* inputTime) + RTC_LOCKS_EXCLUDED(_critSect); OSStatus implInConverterProc(UInt32* numberDataPackets, AudioBufferList* data); diff --git a/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc b/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc index e2b7d6370d9..b34b5c34226 100644 --- a/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc +++ b/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.cc @@ -63,16 +63,19 @@ int32_t AudioMixerManagerMac::Close() { rtc::CritScope lock(&_critSect); - CloseSpeaker(); - CloseMicrophone(); + CloseSpeakerLocked(); + CloseMicrophoneLocked(); return 0; } int32_t AudioMixerManagerMac::CloseSpeaker() { - RTC_LOG(LS_VERBOSE) << __FUNCTION__; - rtc::CritScope lock(&_critSect); + return CloseSpeakerLocked(); +} + +int32_t AudioMixerManagerMac::CloseSpeakerLocked() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; _outputDeviceID = kAudioObjectUnknown; _noOutputChannels = 0; @@ -81,9 +84,12 @@ int32_t AudioMixerManagerMac::CloseSpeaker() { } int32_t AudioMixerManagerMac::CloseMicrophone() { - RTC_LOG(LS_VERBOSE) << __FUNCTION__; - rtc::CritScope lock(&_critSect); + return CloseMicrophoneLocked(); +} + +int32_t AudioMixerManagerMac::CloseMicrophoneLocked() { + RTC_LOG(LS_VERBOSE) << __FUNCTION__; _inputDeviceID = kAudioObjectUnknown; _noInputChannels = 0; diff --git a/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h b/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h index 342e1c997c0..ee6149d672f 100644 --- a/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h +++ b/chromium/third_party/webrtc/modules/audio_device/mac/audio_mixer_manager_mac.h @@ -21,29 +21,29 @@ namespace webrtc { class AudioMixerManagerMac { public: - int32_t OpenSpeaker(AudioDeviceID deviceID); - int32_t OpenMicrophone(AudioDeviceID deviceID); - int32_t SetSpeakerVolume(uint32_t volume); + int32_t OpenSpeaker(AudioDeviceID deviceID) RTC_LOCKS_EXCLUDED(_critSect); + int32_t OpenMicrophone(AudioDeviceID deviceID) RTC_LOCKS_EXCLUDED(_critSect); + int32_t SetSpeakerVolume(uint32_t volume) RTC_LOCKS_EXCLUDED(_critSect); int32_t SpeakerVolume(uint32_t& volume) const; int32_t MaxSpeakerVolume(uint32_t& maxVolume) const; int32_t MinSpeakerVolume(uint32_t& minVolume) const; int32_t SpeakerVolumeIsAvailable(bool& available); int32_t SpeakerMuteIsAvailable(bool& available); - int32_t SetSpeakerMute(bool enable); + int32_t SetSpeakerMute(bool enable) RTC_LOCKS_EXCLUDED(_critSect); int32_t SpeakerMute(bool& enabled) const; int32_t StereoPlayoutIsAvailable(bool& available); int32_t StereoRecordingIsAvailable(bool& available); int32_t MicrophoneMuteIsAvailable(bool& available); - int32_t SetMicrophoneMute(bool enable); + int32_t SetMicrophoneMute(bool enable) RTC_LOCKS_EXCLUDED(_critSect); int32_t MicrophoneMute(bool& enabled) const; int32_t MicrophoneVolumeIsAvailable(bool& available); - int32_t SetMicrophoneVolume(uint32_t volume); + int32_t SetMicrophoneVolume(uint32_t volume) RTC_LOCKS_EXCLUDED(_critSect); int32_t MicrophoneVolume(uint32_t& volume) const; int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const; int32_t MinMicrophoneVolume(uint32_t& minVolume) const; - int32_t Close(); - int32_t CloseSpeaker(); - int32_t CloseMicrophone(); + int32_t Close() RTC_LOCKS_EXCLUDED(_critSect); + int32_t CloseSpeaker() RTC_LOCKS_EXCLUDED(_critSect); + int32_t CloseMicrophone() RTC_LOCKS_EXCLUDED(_critSect); bool SpeakerIsInitialized() const; bool MicrophoneIsInitialized() const; @@ -52,6 +52,8 @@ class AudioMixerManagerMac { ~AudioMixerManagerMac(); private: + int32_t CloseSpeakerLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(_critSect); + int32_t CloseMicrophoneLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(_critSect); static void logCAMsg(const rtc::LoggingSeverity sev, const char* msg, const char* err); diff --git a/chromium/third_party/webrtc/modules/audio_device/mock_audio_device_buffer.h b/chromium/third_party/webrtc/modules/audio_device/mock_audio_device_buffer.h index 1f809cc5dcb..b0f54c20ffa 100644 --- a/chromium/third_party/webrtc/modules/audio_device/mock_audio_device_buffer.h +++ b/chromium/third_party/webrtc/modules/audio_device/mock_audio_device_buffer.h @@ -20,12 +20,14 @@ class MockAudioDeviceBuffer : public AudioDeviceBuffer { public: using AudioDeviceBuffer::AudioDeviceBuffer; virtual ~MockAudioDeviceBuffer() {} - MOCK_METHOD1(RequestPlayoutData, int32_t(size_t nSamples)); - MOCK_METHOD1(GetPlayoutData, int32_t(void* audioBuffer)); - MOCK_METHOD2(SetRecordedBuffer, - int32_t(const void* audioBuffer, size_t nSamples)); - MOCK_METHOD2(SetVQEData, void(int playDelayMS, int recDelayMS)); - MOCK_METHOD0(DeliverRecordedData, int32_t()); + MOCK_METHOD(int32_t, RequestPlayoutData, (size_t nSamples), (override)); + MOCK_METHOD(int32_t, GetPlayoutData, (void* audioBuffer), (override)); + MOCK_METHOD(int32_t, + SetRecordedBuffer, + (const void* audioBuffer, size_t nSamples), + (override)); + MOCK_METHOD(void, SetVQEData, (int playDelayMS, int recDelayMS), (override)); + MOCK_METHOD(int32_t, DeliverRecordedData, (), (override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc b/chromium/third_party/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc index f899dd618a5..383771ce605 100644 --- a/chromium/third_party/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_mixer/audio_mixer_impl_unittest.cc @@ -74,11 +74,13 @@ class MockMixerAudioSource : public ::testing::NiceMock<AudioMixer::Source> { .WillByDefault(Return(kDefaultSampleRateHz)); } - MOCK_METHOD2(GetAudioFrameWithInfo, - AudioFrameInfo(int sample_rate_hz, AudioFrame* audio_frame)); + MOCK_METHOD(AudioFrameInfo, + GetAudioFrameWithInfo, + (int sample_rate_hz, AudioFrame* audio_frame), + (override)); - MOCK_CONST_METHOD0(PreferredSampleRate, int()); - MOCK_CONST_METHOD0(Ssrc, int()); + MOCK_METHOD(int, PreferredSampleRate, (), (const, override)); + MOCK_METHOD(int, Ssrc, (), (const, override)); AudioFrame* fake_frame() { return &fake_frame_; } AudioFrameInfo fake_info() { return fake_audio_frame_info_; } @@ -604,7 +606,7 @@ class HighOutputRateCalculator : public OutputRateCalculator { }; const int HighOutputRateCalculator::kDefaultFrequency; -TEST(AudioMixer, MultipleChannelsAndHighRate) { +TEST(AudioMixerDeathTest, MultipleChannelsAndHighRate) { constexpr size_t kSamplesPerChannel = HighOutputRateCalculator::kDefaultFrequency / 100; // As many channels as an AudioFrame can fit: diff --git a/chromium/third_party/webrtc/modules/audio_mixer/frame_combiner_unittest.cc b/chromium/third_party/webrtc/modules/audio_mixer/frame_combiner_unittest.cc index 5f024a4a556..4b189a052e3 100644 --- a/chromium/third_party/webrtc/modules/audio_mixer/frame_combiner_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_mixer/frame_combiner_unittest.cc @@ -89,7 +89,7 @@ TEST(FrameCombiner, BasicApiCallsLimiter) { } // There are DCHECKs in place to check for invalid parameters. -TEST(FrameCombiner, DebugBuildCrashesWithManyChannels) { +TEST(FrameCombinerDeathTest, DebugBuildCrashesWithManyChannels) { FrameCombiner combiner(true); for (const int rate : {8000, 18000, 34000, 48000}) { for (const int number_of_channels : {10, 20, 21}) { @@ -118,7 +118,7 @@ TEST(FrameCombiner, DebugBuildCrashesWithManyChannels) { } } -TEST(FrameCombiner, DebugBuildCrashesWithHighRate) { +TEST(FrameCombinerDeathTest, DebugBuildCrashesWithHighRate) { FrameCombiner combiner(true); for (const int rate : {50000, 96000, 128000, 196000}) { for (const int number_of_channels : {1, 2, 3}) { diff --git a/chromium/third_party/webrtc/modules/audio_processing/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/BUILD.gn index 7ca78e20b4f..22e128da99d 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/BUILD.gn @@ -53,8 +53,8 @@ rtc_library("api") { "../../rtc_base/system:file_wrapper", "../../rtc_base/system:rtc_export", "agc:gain_control_interface", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("audio_frame_proxies") { @@ -203,8 +203,8 @@ rtc_library("audio_processing") { "ns", "transient:transient_suppressor_api", "vad", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] deps += [ "../../common_audio", @@ -255,8 +255,8 @@ rtc_source_set("rms_level") { deps = [ "../../api:array_view", "../../rtc_base:checks", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("audio_processing_statistics") { @@ -265,10 +265,8 @@ rtc_library("audio_processing_statistics") { "include/audio_processing_statistics.cc", "include/audio_processing_statistics.h", ] - deps = [ - "../../rtc_base/system:rtc_export", - "//third_party/abseil-cpp/absl/types:optional", - ] + deps = [ "../../rtc_base/system:rtc_export" ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_source_set("audio_frame_view") { @@ -395,8 +393,8 @@ if (rtc_include_tests) { "utility:pffft_wrapper_unittest", "vad:vad_unittests", "//testing/gtest", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] defines = [] @@ -422,8 +420,8 @@ if (rtc_include_tests) { "../../rtc_base:rtc_task_queue", "aec_dump", "aec_dump:aec_dump_unittests", - "//third_party/abseil-cpp/absl/flags:flag", ] + absl_deps += [ "//third_party/abseil-cpp/absl/flags:flag" ] sources += [ "audio_processing_impl_locking_unittest.cc", "audio_processing_impl_unittest.cc", @@ -481,8 +479,8 @@ if (rtc_include_tests) { "../../rtc_base:rtc_base_approved", "../../rtc_base:safe_minmax", "agc:gain_map", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } if (rtc_enable_protobuf) { @@ -527,6 +525,8 @@ if (rtc_include_tests) { "aec_dump", "aec_dump:aec_dump_impl", "//testing/gtest", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/flags:parse", "//third_party/abseil-cpp/absl/strings", @@ -609,6 +609,6 @@ rtc_library("audioproc_test_utils") { "../../test:test_support", "../audio_coding:neteq_input_audio_tools", "//testing/gtest", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/aec3/BUILD.gn index d07ffa6abe4..507f2bc8bda 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/BUILD.gn @@ -150,8 +150,8 @@ rtc_library("aec3") { "../../../system_wrappers:field_trial", "../../../system_wrappers:metrics", "../utility:cascaded_biquad_filter", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } if (rtc_include_tests) { @@ -187,8 +187,8 @@ if (rtc_include_tests) { "../../../test:field_trial", "../../../test:test_support", "../utility:cascaded_biquad_filter", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] defines = [] diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc index 8e4f5d96443..39f4e111928 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/adaptive_fir_filter_unittest.cc @@ -285,13 +285,13 @@ TEST_P(AdaptiveFirFilterOneTwoFourEightRenderChannels, #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non-null data dumper works. -TEST(AdaptiveFirFilterTest, NullDataDumper) { +TEST(AdaptiveFirFilterDeathTest, NullDataDumper) { EXPECT_DEATH(AdaptiveFirFilter(9, 9, 250, 1, DetectOptimization(), nullptr), ""); } // Verifies that the check for non-null filter output works. -TEST(AdaptiveFirFilterTest, NullFilterOutput) { +TEST(AdaptiveFirFilterDeathTest, NullFilterOutput) { ApmDataDumper data_dumper(42); AdaptiveFirFilter filter(9, 9, 250, 1, DetectOptimization(), &data_dumper); std::unique_ptr<RenderDelayBuffer> render_delay_buffer( diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.cc index 18321018552..d1d4f7da067 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.cc @@ -15,6 +15,7 @@ #include <iterator> #include "rtc_base/checks.h" +#include "system_wrappers/include/cpu_features_wrapper.h" namespace webrtc { @@ -70,8 +71,18 @@ const float kSqrtHanning128[kFftLength] = { 0.19509032201613f, 0.17096188876030f, 0.14673047445536f, 0.12241067519922f, 0.09801714032956f, 0.07356456359967f, 0.04906767432742f, 0.02454122852291f}; +bool IsSse2Available() { +#if defined(WEBRTC_ARCH_X86_FAMILY) + return WebRtc_GetCPUInfo(kSSE2) != 0; +#else + return false; +#endif +} + } // namespace +Aec3Fft::Aec3Fft() : ooura_fft_(IsSse2Available()) {} + // TODO(peah): Change x to be std::array once the rest of the code allows this. void Aec3Fft::ZeroPaddedFft(rtc::ArrayView<const float> x, Window window, diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.h index 7a2e024d759..6f7fbe4d0e1 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft.h @@ -28,7 +28,8 @@ class Aec3Fft { public: enum class Window { kRectangular, kHanning, kSqrtHanning }; - Aec3Fft() = default; + Aec3Fft(); + // Computes the FFT. Note that both the input and output are modified. void Fft(std::array<float, kFftLength>* x, FftData* X) const { RTC_DCHECK(x); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc index 82d6e766cc8..e60ef5b7132 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/aec3_fft_unittest.cc @@ -20,28 +20,28 @@ namespace webrtc { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non-null input in Fft works. -TEST(Aec3Fft, NullFftInput) { +TEST(Aec3FftDeathTest, NullFftInput) { Aec3Fft fft; FftData X; EXPECT_DEATH(fft.Fft(nullptr, &X), ""); } // Verifies that the check for non-null input in Fft works. -TEST(Aec3Fft, NullFftOutput) { +TEST(Aec3FftDeathTest, NullFftOutput) { Aec3Fft fft; std::array<float, kFftLength> x; EXPECT_DEATH(fft.Fft(&x, nullptr), ""); } // Verifies that the check for non-null output in Ifft works. -TEST(Aec3Fft, NullIfftOutput) { +TEST(Aec3FftDeathTest, NullIfftOutput) { Aec3Fft fft; FftData X; EXPECT_DEATH(fft.Ifft(X, nullptr), ""); } // Verifies that the check for non-null output in ZeroPaddedFft works. -TEST(Aec3Fft, NullZeroPaddedFftOutput) { +TEST(Aec3FftDeathTest, NullZeroPaddedFftOutput) { Aec3Fft fft; std::array<float, kFftLengthBy2> x; EXPECT_DEATH(fft.ZeroPaddedFft(x, Aec3Fft::Window::kRectangular, nullptr), @@ -49,7 +49,7 @@ TEST(Aec3Fft, NullZeroPaddedFftOutput) { } // Verifies that the check for input length in ZeroPaddedFft works. -TEST(Aec3Fft, ZeroPaddedFftWrongInputLength) { +TEST(Aec3FftDeathTest, ZeroPaddedFftWrongInputLength) { Aec3Fft fft; FftData X; std::array<float, kFftLengthBy2 - 1> x; @@ -57,7 +57,7 @@ TEST(Aec3Fft, ZeroPaddedFftWrongInputLength) { } // Verifies that the check for non-null output in PaddedFft works. -TEST(Aec3Fft, NullPaddedFftOutput) { +TEST(Aec3FftDeathTest, NullPaddedFftOutput) { Aec3Fft fft; std::array<float, kFftLengthBy2> x; std::array<float, kFftLengthBy2> x_old; @@ -65,7 +65,7 @@ TEST(Aec3Fft, NullPaddedFftOutput) { } // Verifies that the check for input length in PaddedFft works. -TEST(Aec3Fft, PaddedFftWrongInputLength) { +TEST(Aec3FftDeathTest, PaddedFftWrongInputLength) { Aec3Fft fft; FftData X; std::array<float, kFftLengthBy2 - 1> x; @@ -74,7 +74,7 @@ TEST(Aec3Fft, PaddedFftWrongInputLength) { } // Verifies that the check for length in the old value in PaddedFft works. -TEST(Aec3Fft, PaddedFftWrongOldValuesLength) { +TEST(Aec3FftDeathTest, PaddedFftWrongOldValuesLength) { Aec3Fft fft; FftData X; std::array<float, kFftLengthBy2> x; diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/alignment_mixer_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/alignment_mixer_unittest.cc index 832e4ea8845..03ef06614b5 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/alignment_mixer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/alignment_mixer_unittest.cc @@ -175,7 +175,7 @@ TEST(AlignmentMixer, FixedMode) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST(AlignmentMixer, ZeroNumChannels) { +TEST(AlignmentMixerDeathTest, ZeroNumChannels) { EXPECT_DEATH( AlignmentMixer(/*num_channels*/ 0, /*downmix*/ false, /*adaptive_selection*/ false, /*excitation_limit*/ 1.f, @@ -183,7 +183,7 @@ TEST(AlignmentMixer, ZeroNumChannels) { , ""); } -TEST(AlignmentMixer, IncorrectVariant) { +TEST(AlignmentMixerDeathTest, IncorrectVariant) { EXPECT_DEATH( AlignmentMixer(/*num_channels*/ 1, /*downmix*/ true, /*adaptive_selection*/ true, /*excitation_limit*/ 1.f, diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc index e9a16d06d56..d67967bc02a 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/block_framer_unittest.cc @@ -214,7 +214,8 @@ std::string ProduceDebugText(int sample_rate_hz, size_t num_channels) { } // namespace #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlockAndExtractSubFrame) { +TEST(BlockFramerDeathTest, + WrongNumberOfBandsInBlockForInsertBlockAndExtractSubFrame) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -227,7 +228,7 @@ TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlockAndExtractSubFrame) { } } -TEST(BlockFramer, +TEST(BlockFramerDeathTest, WrongNumberOfChannelsInBlockForInsertBlockAndExtractSubFrame) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { @@ -241,7 +242,7 @@ TEST(BlockFramer, } } -TEST(BlockFramer, +TEST(BlockFramerDeathTest, WrongNumberOfBandsInSubFrameForInsertBlockAndExtractSubFrame) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { @@ -255,7 +256,7 @@ TEST(BlockFramer, } } -TEST(BlockFramer, +TEST(BlockFramerDeathTest, WrongNumberOfChannelsInSubFrameForInsertBlockAndExtractSubFrame) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { @@ -269,7 +270,8 @@ TEST(BlockFramer, } } -TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlockAndExtractSubFrame) { +TEST(BlockFramerDeathTest, + WrongNumberOfSamplesInBlockForInsertBlockAndExtractSubFrame) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -282,7 +284,7 @@ TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlockAndExtractSubFrame) { } } -TEST(BlockFramer, +TEST(BlockFramerDeathTest, WrongNumberOfSamplesInSubFrameForInsertBlockAndExtractSubFrame) { const size_t correct_num_channels = 1; for (auto rate : {16000, 32000, 48000}) { @@ -295,7 +297,7 @@ TEST(BlockFramer, } } -TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlock) { +TEST(BlockFramerDeathTest, WrongNumberOfBandsInBlockForInsertBlock) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -308,7 +310,7 @@ TEST(BlockFramer, WrongNumberOfBandsInBlockForInsertBlock) { } } -TEST(BlockFramer, WrongNumberOfChannelsInBlockForInsertBlock) { +TEST(BlockFramerDeathTest, WrongNumberOfChannelsInBlockForInsertBlock) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -321,7 +323,7 @@ TEST(BlockFramer, WrongNumberOfChannelsInBlockForInsertBlock) { } } -TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlock) { +TEST(BlockFramerDeathTest, WrongNumberOfSamplesInBlockForInsertBlock) { for (auto rate : {16000, 32000, 48000}) { for (auto correct_num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -333,7 +335,7 @@ TEST(BlockFramer, WrongNumberOfSamplesInBlockForInsertBlock) { } } -TEST(BlockFramer, WrongNumberOfPreceedingApiCallsForInsertBlock) { +TEST(BlockFramerDeathTest, WrongNumberOfPreceedingApiCallsForInsertBlock) { for (size_t num_channels : {1, 2, 8}) { for (auto rate : {16000, 32000, 48000}) { for (size_t num_calls = 0; num_calls < 4; ++num_calls) { @@ -351,17 +353,17 @@ TEST(BlockFramer, WrongNumberOfPreceedingApiCallsForInsertBlock) { } // Verifies that the verification for 0 number of channels works. -TEST(BlockFramer, ZeroNumberOfChannelsParameter) { +TEST(BlockFramerDeathTest, ZeroNumberOfChannelsParameter) { EXPECT_DEATH(BlockFramer(16000, 0), ""); } // Verifies that the verification for 0 number of bands works. -TEST(BlockFramer, ZeroNumberOfBandsParameter) { +TEST(BlockFramerDeathTest, ZeroNumberOfBandsParameter) { EXPECT_DEATH(BlockFramer(0, 1), ""); } // Verifies that the verification for null sub_frame pointer works. -TEST(BlockFramer, NullSubFrameParameter) { +TEST(BlockFramerDeathTest, NullSubFrameParameter) { EXPECT_DEATH(BlockFramer(1, 1).InsertBlockAndExtractSubFrame( std::vector<std::vector<std::vector<float>>>( 1, std::vector<std::vector<float>>( diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc index 2b928e877b1..911dad4c818 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/block_processor_unittest.cc @@ -252,21 +252,21 @@ TEST(BlockProcessor, TestLongerCall) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // TODO(gustaf): Re-enable the test once the issue with memory leaks during // DEATH tests on test bots has been fixed. -TEST(BlockProcessor, DISABLED_VerifyRenderBlockSizeCheck) { +TEST(BlockProcessorDeathTest, DISABLED_VerifyRenderBlockSizeCheck) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunRenderBlockSizeVerificationTest(rate); } } -TEST(BlockProcessor, VerifyCaptureBlockSizeCheck) { +TEST(BlockProcessorDeathTest, VerifyCaptureBlockSizeCheck) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunCaptureBlockSizeVerificationTest(rate); } } -TEST(BlockProcessor, VerifyRenderNumBandsCheck) { +TEST(BlockProcessorDeathTest, VerifyRenderNumBandsCheck) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunRenderNumBandsVerificationTest(rate); @@ -275,7 +275,7 @@ TEST(BlockProcessor, VerifyRenderNumBandsCheck) { // TODO(peah): Verify the check for correct number of bands in the capture // signal. -TEST(BlockProcessor, VerifyCaptureNumBandsCheck) { +TEST(BlockProcessorDeathTest, VerifyCaptureNumBandsCheck) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); RunCaptureNumBandsVerificationTest(rate); @@ -283,7 +283,7 @@ TEST(BlockProcessor, VerifyCaptureNumBandsCheck) { } // Verifiers that the verification for null ProcessCapture input works. -TEST(BlockProcessor, NullProcessCaptureParameter) { +TEST(BlockProcessorDeathTest, NullProcessCaptureParameter) { EXPECT_DEATH(std::unique_ptr<BlockProcessor>( BlockProcessor::Create(EchoCanceller3Config(), 16000, 1, 1)) ->ProcessCapture(false, false, nullptr, nullptr), diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/clockdrift_detector.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/clockdrift_detector.h index 22528c94892..2ba90bb8890 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/clockdrift_detector.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/clockdrift_detector.h @@ -11,6 +11,8 @@ #ifndef MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ #define MODULES_AUDIO_PROCESSING_AEC3_CLOCKDRIFT_DETECTOR_H_ +#include <stddef.h> + #include <array> namespace webrtc { diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/coarse_filter_update_gain_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/coarse_filter_update_gain_unittest.cc index 4185c1adb81..92775cf7028 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/coarse_filter_update_gain_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/coarse_filter_update_gain_unittest.cc @@ -138,7 +138,7 @@ std::string ProduceDebugText(size_t delay, int filter_length_blocks) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non-null output gain parameter works. -TEST(CoarseFilterUpdateGain, NullDataOutputGain) { +TEST(CoarseFilterUpdateGainDeathTest, NullDataOutputGain) { ApmDataDumper data_dumper(42); FftBuffer fft_buffer(1, 1); RenderSignalAnalyzer analyzer(EchoCanceller3Config{}); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/decimator_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/decimator_unittest.cc index 1e279cea3e5..e6f5ea04034 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/decimator_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/decimator_unittest.cc @@ -103,7 +103,7 @@ TEST(Decimator, NoLeakageFromUpperFrequencies) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for the input size. -TEST(Decimator, WrongInputSize) { +TEST(DecimatorDeathTest, WrongInputSize) { Decimator decimator(4); std::vector<float> x(kBlockSize - 1, 0.f); std::array<float, kBlockSize / 4> x_downsampled; @@ -111,14 +111,14 @@ TEST(Decimator, WrongInputSize) { } // Verifies the check for non-null output parameter. -TEST(Decimator, NullOutput) { +TEST(DecimatorDeathTest, NullOutput) { Decimator decimator(4); std::vector<float> x(kBlockSize, 0.f); EXPECT_DEATH(decimator.Decimate(x, nullptr), ""); } // Verifies the check for the output size. -TEST(Decimator, WrongOutputSize) { +TEST(DecimatorDeathTest, WrongOutputSize) { Decimator decimator(4); std::vector<float> x(kBlockSize, 0.f); std::array<float, kBlockSize / 4 - 1> x_downsampled; @@ -126,7 +126,7 @@ TEST(Decimator, WrongOutputSize) { } // Verifies the check for the correct downsampling factor. -TEST(Decimator, CorrectDownSamplingFactor) { +TEST(DecimatorDeathTest, CorrectDownSamplingFactor) { EXPECT_DEATH(Decimator(3), ""); } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc index 21255f192e1..04d93e4db43 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_canceller3_unittest.cc @@ -890,7 +890,7 @@ TEST(EchoCanceller3FieldTrials, Aec3SuppressorTuningOverrideOneParam) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST(EchoCanceller3InputCheck, WrongCaptureNumBandsCheckVerification) { +TEST(EchoCanceller3InputCheckDeathTest, WrongCaptureNumBandsCheckVerification) { for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Tester(rate).RunProcessCaptureNumBandsCheckVerification(); @@ -899,7 +899,7 @@ TEST(EchoCanceller3InputCheck, WrongCaptureNumBandsCheckVerification) { // Verifiers that the verification for null input to the capture processing api // call works. -TEST(EchoCanceller3InputCheck, NullCaptureProcessingParameter) { +TEST(EchoCanceller3InputCheckDeathTest, NullCaptureProcessingParameter) { EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 16000, 1, 1) .ProcessCapture(nullptr, false), ""); @@ -908,7 +908,7 @@ TEST(EchoCanceller3InputCheck, NullCaptureProcessingParameter) { // Verifies the check for correct sample rate. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. -TEST(EchoCanceller3InputCheck, DISABLED_WrongSampleRate) { +TEST(EchoCanceller3InputCheckDeathTest, DISABLED_WrongSampleRate) { ApmDataDumper data_dumper(0); EXPECT_DEATH(EchoCanceller3(EchoCanceller3Config(), 8001, 1, 1), ""); } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc index 8003a11bbcf..6ba4cdd0d7c 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_path_delay_estimator_unittest.cc @@ -165,7 +165,7 @@ TEST(EchoPathDelayEstimator, NoDelayEstimatesForLowLevelRenderSignals) { // Verifies the check for the render blocksize. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. -TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) { +TEST(EchoPathDelayEstimatorDeathTest, DISABLED_WrongRenderBlockSize) { ApmDataDumper data_dumper(0); EchoCanceller3Config config; EchoPathDelayEstimator estimator(&data_dumper, config, 1); @@ -180,7 +180,7 @@ TEST(EchoPathDelayEstimator, DISABLED_WrongRenderBlockSize) { // Verifies the check for the capture blocksize. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. -TEST(EchoPathDelayEstimator, WrongCaptureBlockSize) { +TEST(EchoPathDelayEstimatorDeathTest, WrongCaptureBlockSize) { ApmDataDumper data_dumper(0); EchoCanceller3Config config; EchoPathDelayEstimator estimator(&data_dumper, config, 1); @@ -194,7 +194,7 @@ TEST(EchoPathDelayEstimator, WrongCaptureBlockSize) { } // Verifies the check for non-null data dumper. -TEST(EchoPathDelayEstimator, NullDataDumper) { +TEST(EchoPathDelayEstimatorDeathTest, NullDataDumper) { EXPECT_DEATH(EchoPathDelayEstimator(nullptr, EchoCanceller3Config(), 1), ""); } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc index 30c66118699..45b30a9c74c 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_metrics_unittest.cc @@ -23,7 +23,7 @@ namespace webrtc { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for non-null input. -TEST(UpdateDbMetric, NullValue) { +TEST(UpdateDbMetricDeathTest, NullValue) { std::array<float, kFftLengthBy2Plus1> value; value.fill(0.f); EXPECT_DEATH(aec3::UpdateDbMetric(value, nullptr), ""); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc index e050027c63f..77a207659ce 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/echo_remover_unittest.cc @@ -91,14 +91,14 @@ TEST_P(EchoRemoverMultiChannel, BasicApiCalls) { // Verifies the check for the samplerate. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. -TEST(EchoRemover, DISABLED_WrongSampleRate) { +TEST(EchoRemoverDeathTest, DISABLED_WrongSampleRate) { EXPECT_DEATH(std::unique_ptr<EchoRemover>( EchoRemover::Create(EchoCanceller3Config(), 8001, 1, 1)), ""); } // Verifies the check for the capture block size. -TEST(EchoRemover, WrongCaptureBlockSize) { +TEST(EchoRemoverDeathTest, WrongCaptureBlockSize) { absl::optional<DelayEstimate> delay_estimate; for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); @@ -121,7 +121,7 @@ TEST(EchoRemover, WrongCaptureBlockSize) { // Verifies the check for the number of capture bands. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed.c -TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { +TEST(EchoRemoverDeathTest, DISABLED_WrongCaptureNumBands) { absl::optional<DelayEstimate> delay_estimate; for (auto rate : {16000, 32000, 48000}) { SCOPED_TRACE(ProduceDebugText(rate)); @@ -143,7 +143,7 @@ TEST(EchoRemover, DISABLED_WrongCaptureNumBands) { } // Verifies the check for non-null capture block. -TEST(EchoRemover, NullCapture) { +TEST(EchoRemoverDeathTest, NullCapture) { absl::optional<DelayEstimate> delay_estimate; std::unique_ptr<EchoRemover> remover( EchoRemover::Create(EchoCanceller3Config(), 16000, 1, 1)); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc index 0812fd64208..9be26804533 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/fft_data_unittest.cc @@ -44,12 +44,12 @@ TEST(FftData, TestOptimizations) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for null output in CopyToPackedArray. -TEST(FftData, NonNullCopyToPackedArrayOutput) { +TEST(FftDataDeathTest, NonNullCopyToPackedArrayOutput) { EXPECT_DEATH(FftData().CopyToPackedArray(nullptr), ""); } // Verifies the check for null output in Spectrum. -TEST(FftData, NonNullSpectrumOutput) { +TEST(FftDataDeathTest, NonNullSpectrumOutput) { EXPECT_DEATH(FftData().Spectrum(Aec3Optimization::kNone, nullptr), ""); } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc index e907608d95f..216f5150377 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/frame_blocker_unittest.cc @@ -287,7 +287,8 @@ std::string ProduceDebugText(int sample_rate_hz, size_t num_channels) { } // namespace #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST(FrameBlocker, WrongNumberOfBandsInBlockForInsertSubFrameAndExtractBlock) { +TEST(FrameBlockerDeathTest, + WrongNumberOfBandsInBlockForInsertSubFrameAndExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -300,7 +301,7 @@ TEST(FrameBlocker, WrongNumberOfBandsInBlockForInsertSubFrameAndExtractBlock) { } } -TEST(FrameBlocker, +TEST(FrameBlockerDeathTest, WrongNumberOfChannelsInBlockForInsertSubFrameAndExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { @@ -314,7 +315,7 @@ TEST(FrameBlocker, } } -TEST(FrameBlocker, +TEST(FrameBlockerDeathTest, WrongNumberOfBandsInSubFrameForInsertSubFrameAndExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { @@ -328,7 +329,7 @@ TEST(FrameBlocker, } } -TEST(FrameBlocker, +TEST(FrameBlockerDeathTest, WrongNumberOfChannelsInSubFrameForInsertSubFrameAndExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { @@ -342,7 +343,7 @@ TEST(FrameBlocker, } } -TEST(FrameBlocker, +TEST(FrameBlockerDeathTest, WrongNumberOfSamplesInBlockForInsertSubFrameAndExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { @@ -356,7 +357,7 @@ TEST(FrameBlocker, } } -TEST(FrameBlocker, +TEST(FrameBlockerDeathTest, WrongNumberOfSamplesInSubFrameForInsertSubFrameAndExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { @@ -370,7 +371,7 @@ TEST(FrameBlocker, } } -TEST(FrameBlocker, WrongNumberOfBandsInBlockForExtractBlock) { +TEST(FrameBlockerDeathTest, WrongNumberOfBandsInBlockForExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -383,7 +384,7 @@ TEST(FrameBlocker, WrongNumberOfBandsInBlockForExtractBlock) { } } -TEST(FrameBlocker, WrongNumberOfChannelsInBlockForExtractBlock) { +TEST(FrameBlockerDeathTest, WrongNumberOfChannelsInBlockForExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -396,7 +397,7 @@ TEST(FrameBlocker, WrongNumberOfChannelsInBlockForExtractBlock) { } } -TEST(FrameBlocker, WrongNumberOfSamplesInBlockForExtractBlock) { +TEST(FrameBlockerDeathTest, WrongNumberOfSamplesInBlockForExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t correct_num_channels : {1, 2, 4, 8}) { SCOPED_TRACE(ProduceDebugText(rate, correct_num_channels)); @@ -408,7 +409,7 @@ TEST(FrameBlocker, WrongNumberOfSamplesInBlockForExtractBlock) { } } -TEST(FrameBlocker, WrongNumberOfPreceedingApiCallsForExtractBlock) { +TEST(FrameBlockerDeathTest, WrongNumberOfPreceedingApiCallsForExtractBlock) { for (auto rate : {16000, 32000, 48000}) { for (size_t num_channels : {1, 2, 4, 8}) { for (size_t num_calls = 0; num_calls < 4; ++num_calls) { @@ -426,17 +427,17 @@ TEST(FrameBlocker, WrongNumberOfPreceedingApiCallsForExtractBlock) { } // Verifies that the verification for 0 number of channels works. -TEST(FrameBlocker, ZeroNumberOfChannelsParameter) { +TEST(FrameBlockerDeathTest, ZeroNumberOfChannelsParameter) { EXPECT_DEATH(FrameBlocker(16000, 0), ""); } // Verifies that the verification for 0 number of bands works. -TEST(FrameBlocker, ZeroNumberOfBandsParameter) { +TEST(FrameBlockerDeathTest, ZeroNumberOfBandsParameter) { EXPECT_DEATH(FrameBlocker(0, 1), ""); } // Verifiers that the verification for null sub_frame pointer works. -TEST(FrameBlocker, NullBlockParameter) { +TEST(FrameBlockerDeathTest, NullBlockParameter) { std::vector<std::vector<std::vector<float>>> sub_frame( 1, std::vector<std::vector<float>>( 1, std::vector<float>(kSubFrameLength, 0.f))); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc index e136c898772..8e2a12e6c5d 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator_unittest.cc @@ -144,7 +144,7 @@ TEST(MatchedFilterLagAggregator, DISABLED_PersistentAggregatedLag) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for non-null data dumper. -TEST(MatchedFilterLagAggregator, NullDataDumper) { +TEST(MatchedFilterLagAggregatorDeathTest, NullDataDumper) { EchoCanceller3Config config; EXPECT_DEATH(MatchedFilterLagAggregator( nullptr, 10, config.delay.delay_selection_thresholds), diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc index 8a6e22eecaa..7d9a7d4d0af 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/matched_filter_unittest.cc @@ -375,7 +375,7 @@ TEST(MatchedFilter, NumberOfLagEstimates) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for non-zero windows size. -TEST(MatchedFilter, ZeroWindowSize) { +TEST(MatchedFilterDeathTest, ZeroWindowSize) { ApmDataDumper data_dumper(0); EchoCanceller3Config config; EXPECT_DEATH(MatchedFilter(&data_dumper, DetectOptimization(), 16, 0, 1, 1, @@ -385,7 +385,7 @@ TEST(MatchedFilter, ZeroWindowSize) { } // Verifies the check for non-null data dumper. -TEST(MatchedFilter, NullDataDumper) { +TEST(MatchedFilterDeathTest, NullDataDumper) { EchoCanceller3Config config; EXPECT_DEATH(MatchedFilter(nullptr, DetectOptimization(), 16, 1, 1, 1, 150, config.delay.delay_estimate_smoothing, @@ -395,7 +395,7 @@ TEST(MatchedFilter, NullDataDumper) { // Verifies the check for that the sub block size is a multiple of 4. // TODO(peah): Activate the unittest once the required code has been landed. -TEST(MatchedFilter, DISABLED_BlockSizeMultipleOf4) { +TEST(MatchedFilterDeathTest, DISABLED_BlockSizeMultipleOf4) { ApmDataDumper data_dumper(0); EchoCanceller3Config config; EXPECT_DEATH(MatchedFilter(&data_dumper, DetectOptimization(), 15, 1, 1, 1, @@ -407,7 +407,7 @@ TEST(MatchedFilter, DISABLED_BlockSizeMultipleOf4) { // Verifies the check for that there is an integer number of sub blocks that add // up to a block size. // TODO(peah): Activate the unittest once the required code has been landed. -TEST(MatchedFilter, DISABLED_SubBlockSizeAddsUpToBlockSize) { +TEST(MatchedFilterDeathTest, DISABLED_SubBlockSizeAddsUpToBlockSize) { ApmDataDumper data_dumper(0); EchoCanceller3Config config; EXPECT_DEATH(MatchedFilter(&data_dumper, DetectOptimization(), 12, 1, 1, 1, diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h index e9a95c837d2..e1eb26702f1 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_block_processor.h @@ -24,17 +24,26 @@ class MockBlockProcessor : public BlockProcessor { MockBlockProcessor(); virtual ~MockBlockProcessor(); - MOCK_METHOD4( - ProcessCapture, - void(bool level_change, - bool saturated_microphone_signal, - std::vector<std::vector<std::vector<float>>>* linear_output, - std::vector<std::vector<std::vector<float>>>* capture_block)); - MOCK_METHOD1(BufferRender, - void(const std::vector<std::vector<std::vector<float>>>& block)); - MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); - MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); - MOCK_METHOD1(SetAudioBufferDelay, void(int delay_ms)); + MOCK_METHOD(void, + ProcessCapture, + (bool level_change, + bool saturated_microphone_signal, + std::vector<std::vector<std::vector<float>>>* linear_output, + std::vector<std::vector<std::vector<float>>>* capture_block), + (override)); + MOCK_METHOD(void, + BufferRender, + (const std::vector<std::vector<std::vector<float>>>& block), + (override)); + MOCK_METHOD(void, + UpdateEchoLeakageStatus, + (bool leakage_detected), + (override)); + MOCK_METHOD(void, + GetMetrics, + (EchoControl::Metrics * metrics), + (const, override)); + MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override)); }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h index 6c580f3a91d..8a3044bcf11 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_echo_remover.h @@ -27,16 +27,23 @@ class MockEchoRemover : public EchoRemover { MockEchoRemover(); virtual ~MockEchoRemover(); - MOCK_METHOD6(ProcessCapture, - void(EchoPathVariability echo_path_variability, - bool capture_signal_saturation, - const absl::optional<DelayEstimate>& delay_estimate, - RenderBuffer* render_buffer, - std::vector<std::vector<std::vector<float>>>* linear_output, - std::vector<std::vector<std::vector<float>>>* capture)); - MOCK_CONST_METHOD0(Delay, absl::optional<int>()); - MOCK_METHOD1(UpdateEchoLeakageStatus, void(bool leakage_detected)); - MOCK_CONST_METHOD1(GetMetrics, void(EchoControl::Metrics* metrics)); + MOCK_METHOD(void, + ProcessCapture, + (EchoPathVariability echo_path_variability, + bool capture_signal_saturation, + const absl::optional<DelayEstimate>& delay_estimate, + RenderBuffer* render_buffer, + std::vector<std::vector<std::vector<float>>>* linear_output, + std::vector<std::vector<std::vector<float>>>* capture), + (override)); + MOCK_METHOD(void, + UpdateEchoLeakageStatus, + (bool leakage_detected), + (override)); + MOCK_METHOD(void, + GetMetrics, + (EchoControl::Metrics * metrics), + (const, override)); }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h index f83c6706327..26f58cfe1ec 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_buffer.h @@ -27,21 +27,26 @@ class MockRenderDelayBuffer : public RenderDelayBuffer { MockRenderDelayBuffer(int sample_rate_hz, size_t num_channels); virtual ~MockRenderDelayBuffer(); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD1(Insert, - RenderDelayBuffer::BufferingEvent( - const std::vector<std::vector<std::vector<float>>>& block)); - MOCK_METHOD0(PrepareCaptureProcessing, RenderDelayBuffer::BufferingEvent()); - MOCK_METHOD1(AlignFromDelay, bool(size_t delay)); - MOCK_METHOD0(AlignFromExternalDelay, void()); - MOCK_CONST_METHOD0(Delay, size_t()); - MOCK_CONST_METHOD0(MaxDelay, size_t()); - MOCK_METHOD0(GetRenderBuffer, RenderBuffer*()); - MOCK_CONST_METHOD0(GetDownsampledRenderBuffer, - const DownsampledRenderBuffer&()); - MOCK_CONST_METHOD1(CausalDelay, bool(size_t delay)); - MOCK_METHOD1(SetAudioBufferDelay, void(int delay_ms)); - MOCK_METHOD0(HasReceivedBufferDelay, bool()); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(RenderDelayBuffer::BufferingEvent, + Insert, + (const std::vector<std::vector<std::vector<float>>>& block), + (override)); + MOCK_METHOD(RenderDelayBuffer::BufferingEvent, + PrepareCaptureProcessing, + (), + (override)); + MOCK_METHOD(bool, AlignFromDelay, (size_t delay), (override)); + MOCK_METHOD(void, AlignFromExternalDelay, (), (override)); + MOCK_METHOD(size_t, Delay, (), (const, override)); + MOCK_METHOD(size_t, MaxDelay, (), (const, override)); + MOCK_METHOD(RenderBuffer*, GetRenderBuffer, (), (override)); + MOCK_METHOD(const DownsampledRenderBuffer&, + GetDownsampledRenderBuffer, + (), + (const, override)); + MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, HasReceivedBufferDelay, (), (override)); private: RenderBuffer* FakeGetRenderBuffer() { return &render_buffer_; } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h index e72333eaeb3..67d8baefe6a 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/mock/mock_render_delay_controller.h @@ -25,14 +25,15 @@ class MockRenderDelayController : public RenderDelayController { MockRenderDelayController(); virtual ~MockRenderDelayController(); - MOCK_METHOD1(Reset, void(bool reset_delay_statistics)); - MOCK_METHOD0(LogRenderCall, void()); - MOCK_METHOD3(GetDelay, - absl::optional<DelayEstimate>( - const DownsampledRenderBuffer& render_buffer, - size_t render_delay_buffer_delay, - const std::vector<std::vector<float>>& capture)); - MOCK_CONST_METHOD0(HasClockdrift, bool()); + MOCK_METHOD(void, Reset, (bool reset_delay_statistics), (override)); + MOCK_METHOD(void, LogRenderCall, (), (override)); + MOCK_METHOD(absl::optional<DelayEstimate>, + GetDelay, + (const DownsampledRenderBuffer& render_buffer, + size_t render_delay_buffer_delay, + const std::vector<std::vector<float>>& capture), + (override)); + MOCK_METHOD(bool, HasClockdrift, (), (const, override)); }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/refined_filter_update_gain_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/refined_filter_update_gain_unittest.cc index 117f34508ee..2393fddd6fa 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/refined_filter_update_gain_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/refined_filter_update_gain_unittest.cc @@ -234,7 +234,7 @@ std::string ProduceDebugText(size_t delay, int filter_length_blocks) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non-null output gain parameter works. -TEST(RefinedFilterUpdateGain, NullDataOutputGain) { +TEST(RefinedFilterUpdateGainDeathTest, NullDataOutputGain) { ApmDataDumper data_dumper(42); EchoCanceller3Config config; RenderSignalAnalyzer analyzer(config); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc index 6981f6d5108..45595286009 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_buffer_unittest.cc @@ -21,21 +21,21 @@ namespace webrtc { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for non-null fft buffer. -TEST(RenderBuffer, NullExternalFftBuffer) { +TEST(RenderBufferDeathTest, NullExternalFftBuffer) { BlockBuffer block_buffer(10, 3, 1, kBlockSize); SpectrumBuffer spectrum_buffer(10, 1); EXPECT_DEATH(RenderBuffer(&block_buffer, &spectrum_buffer, nullptr), ""); } // Verifies the check for non-null spectrum buffer. -TEST(RenderBuffer, NullExternalSpectrumBuffer) { +TEST(RenderBufferDeathTest, NullExternalSpectrumBuffer) { FftBuffer fft_buffer(10, 1); BlockBuffer block_buffer(10, 3, 1, kBlockSize); EXPECT_DEATH(RenderBuffer(&block_buffer, nullptr, &fft_buffer), ""); } // Verifies the check for non-null block buffer. -TEST(RenderBuffer, NullExternalBlockBuffer) { +TEST(RenderBufferDeathTest, NullExternalBlockBuffer) { FftBuffer fft_buffer(10, 1); SpectrumBuffer spectrum_buffer(10, 1); EXPECT_DEATH(RenderBuffer(nullptr, &spectrum_buffer, &fft_buffer), ""); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc index 35e81319cf1..efd4a299206 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_buffer_unittest.cc @@ -97,14 +97,14 @@ TEST(RenderDelayBuffer, AlignFromDelay) { // Verifies the check for feasible delay. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. -TEST(RenderDelayBuffer, DISABLED_WrongDelay) { +TEST(RenderDelayBufferDeathTest, DISABLED_WrongDelay) { std::unique_ptr<RenderDelayBuffer> delay_buffer( RenderDelayBuffer::Create(EchoCanceller3Config(), 48000, 1)); EXPECT_DEATH(delay_buffer->AlignFromDelay(21), ""); } // Verifies the check for the number of bands in the inserted blocks. -TEST(RenderDelayBuffer, WrongNumberOfBands) { +TEST(RenderDelayBufferDeathTest, WrongNumberOfBands) { for (auto rate : {16000, 32000, 48000}) { for (size_t num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate)); @@ -120,7 +120,7 @@ TEST(RenderDelayBuffer, WrongNumberOfBands) { } // Verifies the check for the number of channels in the inserted blocks. -TEST(RenderDelayBuffer, WrongNumberOfChannels) { +TEST(RenderDelayBufferDeathTest, WrongNumberOfChannels) { for (auto rate : {16000, 32000, 48000}) { for (size_t num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate)); @@ -136,7 +136,7 @@ TEST(RenderDelayBuffer, WrongNumberOfChannels) { } // Verifies the check of the length of the inserted blocks. -TEST(RenderDelayBuffer, WrongBlockLength) { +TEST(RenderDelayBufferDeathTest, WrongBlockLength) { for (auto rate : {16000, 32000, 48000}) { for (size_t num_channels : {1, 2, 8}) { SCOPED_TRACE(ProduceDebugText(rate)); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc index fb7b86a75dc..0d3c8564662 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_delay_controller_unittest.cc @@ -325,7 +325,7 @@ TEST(RenderDelayController, DISABLED_AlignmentWithJitter) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for the capture signal block size. -TEST(RenderDelayController, WrongCaptureSize) { +TEST(RenderDelayControllerDeathTest, WrongCaptureSize) { std::vector<std::vector<float>> block( 1, std::vector<float>(kBlockSize - 1, 0.f)); EchoCanceller3Config config; @@ -345,7 +345,7 @@ TEST(RenderDelayController, WrongCaptureSize) { // Verifies the check for correct sample rate. // TODO(peah): Re-enable the test once the issue with memory leaks during DEATH // tests on test bots has been fixed. -TEST(RenderDelayController, DISABLED_WrongSampleRate) { +TEST(RenderDelayControllerDeathTest, DISABLED_WrongSampleRate) { for (auto rate : {-1, 0, 8001, 16001}) { SCOPED_TRACE(ProduceDebugText(rate)); EchoCanceller3Config config; diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc index f40fade8302..7a48cc4b698 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/render_signal_analyzer_unittest.cc @@ -117,7 +117,7 @@ std::string ProduceDebugText(size_t num_channels) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non-null output parameter works. -TEST(RenderSignalAnalyzer, NullMaskOutput) { +TEST(RenderSignalAnalyzerDeathTest, NullMaskOutput) { RenderSignalAnalyzer analyzer(EchoCanceller3Config{}); EXPECT_DEATH(analyzer.MaskRegionsAroundNarrowBands(nullptr), ""); } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc index 72e57879a02..bbc1e4ffc61 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/subtractor_unittest.cc @@ -189,7 +189,7 @@ std::string ProduceDebugText(size_t num_render_channels, #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non data dumper works. -TEST(Subtractor, NullDataDumper) { +TEST(SubtractorDeathTest, NullDataDumper) { EXPECT_DEATH( Subtractor(EchoCanceller3Config(), 1, 1, nullptr, DetectOptimization()), ""); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter.h b/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter.h index 281c2c30c48..dcf2292c7f1 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter.h @@ -14,7 +14,6 @@ #include <array> #include <vector> -#include "common_audio/third_party/ooura/fft_size_128/ooura_fft.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/fft_data.h" @@ -39,7 +38,6 @@ class SuppressionFilter { const Aec3Optimization optimization_; const int sample_rate_hz_; const size_t num_capture_channels_; - const OouraFft ooura_fft_; const Aec3Fft fft_; std::vector<std::vector<std::array<float, kFftLengthBy2>>> e_output_old_; RTC_DISALLOW_COPY_AND_ASSIGN(SuppressionFilter); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc index b55c719fa9f..a160bec0451 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_filter_unittest.cc @@ -50,7 +50,7 @@ void ProduceSinusoid(int sample_rate_hz, #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies the check for null suppressor output. -TEST(SuppressionFilter, NullOutput) { +TEST(SuppressionFilterDeathTest, NullOutput) { std::vector<FftData> cn(1); std::vector<FftData> cn_high_bands(1); std::vector<FftData> E(1); @@ -62,7 +62,7 @@ TEST(SuppressionFilter, NullOutput) { } // Verifies the check for allowed sample rate. -TEST(SuppressionFilter, ProperSampleRate) { +TEST(SuppressionFilterDeathTest, ProperSampleRate) { EXPECT_DEATH(SuppressionFilter(Aec3Optimization::kNone, 16001, 1), ""); } diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc index 0452f2e1fb7..4fb4cd7142b 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/aec3/suppression_gain_unittest.cc @@ -25,7 +25,7 @@ namespace aec3 { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check for non-null output gains works. -TEST(SuppressionGain, NullOutputGains) { +TEST(SuppressionGainDeathTest, NullOutputGains) { std::vector<std::array<float, kFftLengthBy2Plus1>> E2(1, {0.f}); std::vector<std::array<float, kFftLengthBy2Plus1>> R2(1, {0.f}); std::vector<std::array<float, kFftLengthBy2Plus1>> S2(1); diff --git a/chromium/third_party/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h b/chromium/third_party/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h index 65306a7b285..b396739de4c 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h +++ b/chromium/third_party/webrtc/modules/audio_processing/aec_dump/mock_aec_dump.h @@ -25,36 +25,54 @@ class MockAecDump : public AecDump { MockAecDump(); virtual ~MockAecDump(); - MOCK_METHOD2(WriteInitMessage, - void(const ProcessingConfig& api_format, int64_t time_now_ms)); + MOCK_METHOD(void, + WriteInitMessage, + (const ProcessingConfig& api_format, int64_t time_now_ms), + (override)); - MOCK_METHOD1(AddCaptureStreamInput, - void(const AudioFrameView<const float>& src)); - MOCK_METHOD1(AddCaptureStreamOutput, - void(const AudioFrameView<const float>& src)); - MOCK_METHOD3(AddCaptureStreamInput, - void(const int16_t* const data, - int num_channels, - int samples_per_channel)); - MOCK_METHOD3(AddCaptureStreamOutput, - void(const int16_t* const data, - int num_channels, - int samples_per_channel)); - MOCK_METHOD1(AddAudioProcessingState, - void(const AudioProcessingState& state)); - MOCK_METHOD0(WriteCaptureStreamMessage, void()); + MOCK_METHOD(void, + AddCaptureStreamInput, + (const AudioFrameView<const float>& src), + (override)); + MOCK_METHOD(void, + AddCaptureStreamOutput, + (const AudioFrameView<const float>& src), + (override)); + MOCK_METHOD(void, + AddCaptureStreamInput, + (const int16_t* const data, + int num_channels, + int samples_per_channel), + (override)); + MOCK_METHOD(void, + AddCaptureStreamOutput, + (const int16_t* const data, + int num_channels, + int samples_per_channel), + (override)); + MOCK_METHOD(void, + AddAudioProcessingState, + (const AudioProcessingState& state), + (override)); + MOCK_METHOD(void, WriteCaptureStreamMessage, (), (override)); - MOCK_METHOD3(WriteRenderStreamMessage, - void(const int16_t* const data, - int num_channels, - int samples_per_channel)); - MOCK_METHOD1(WriteRenderStreamMessage, - void(const AudioFrameView<const float>& src)); + MOCK_METHOD(void, + WriteRenderStreamMessage, + (const int16_t* const data, + int num_channels, + int samples_per_channel), + (override)); + MOCK_METHOD(void, + WriteRenderStreamMessage, + (const AudioFrameView<const float>& src), + (override)); - MOCK_METHOD1(WriteConfig, void(const InternalAPMConfig& config)); + MOCK_METHOD(void, WriteConfig, (const InternalAPMConfig& config), (override)); - MOCK_METHOD1(WriteRuntimeSetting, - void(const AudioProcessing::RuntimeSetting& config)); + MOCK_METHOD(void, + WriteRuntimeSetting, + (const AudioProcessing::RuntimeSetting& config), + (override)); }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_processing/agc/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/agc/BUILD.gn index 42830c918d8..9ed6399cbf8 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/agc/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/agc/BUILD.gn @@ -36,8 +36,8 @@ rtc_library("agc") { "../../../system_wrappers:metrics", "../agc2:level_estimation_agc", "../vad", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("level_estimation") { diff --git a/chromium/third_party/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc index c5e65adec1b..995801a8cb3 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/agc/agc_manager_direct_unittest.cc @@ -37,22 +37,23 @@ const int kMinMicLevel = 12; class MockGainControl : public GainControl { public: virtual ~MockGainControl() {} - MOCK_METHOD0(Initialize, void()); - MOCK_CONST_METHOD0(is_enabled, bool()); - MOCK_METHOD1(set_stream_analog_level, int(int level)); - MOCK_CONST_METHOD0(stream_analog_level, int()); - MOCK_METHOD1(set_mode, int(Mode mode)); - MOCK_CONST_METHOD0(mode, Mode()); - MOCK_METHOD1(set_target_level_dbfs, int(int level)); - MOCK_CONST_METHOD0(target_level_dbfs, int()); - MOCK_METHOD1(set_compression_gain_db, int(int gain)); - MOCK_CONST_METHOD0(compression_gain_db, int()); - MOCK_METHOD1(enable_limiter, int(bool enable)); - MOCK_CONST_METHOD0(is_limiter_enabled, bool()); - MOCK_METHOD2(set_analog_level_limits, int(int minimum, int maximum)); - MOCK_CONST_METHOD0(analog_level_minimum, int()); - MOCK_CONST_METHOD0(analog_level_maximum, int()); - MOCK_CONST_METHOD0(stream_is_saturated, bool()); + MOCK_METHOD(int, set_stream_analog_level, (int level), (override)); + MOCK_METHOD(int, stream_analog_level, (), (const, override)); + MOCK_METHOD(int, set_mode, (Mode mode), (override)); + MOCK_METHOD(Mode, mode, (), (const, override)); + MOCK_METHOD(int, set_target_level_dbfs, (int level), (override)); + MOCK_METHOD(int, target_level_dbfs, (), (const, override)); + MOCK_METHOD(int, set_compression_gain_db, (int gain), (override)); + MOCK_METHOD(int, compression_gain_db, (), (const, override)); + MOCK_METHOD(int, enable_limiter, (bool enable), (override)); + MOCK_METHOD(bool, is_limiter_enabled, (), (const, override)); + MOCK_METHOD(int, + set_analog_level_limits, + (int minimum, int maximum), + (override)); + MOCK_METHOD(int, analog_level_minimum, (), (const, override)); + MOCK_METHOD(int, analog_level_maximum, (), (const, override)); + MOCK_METHOD(bool, stream_is_saturated, (), (const, override)); }; } // namespace diff --git a/chromium/third_party/webrtc/modules/audio_processing/agc/mock_agc.h b/chromium/third_party/webrtc/modules/audio_processing/agc/mock_agc.h index 6542acc8d54..0ef41c6e52a 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/agc/mock_agc.h +++ b/chromium/third_party/webrtc/modules/audio_processing/agc/mock_agc.h @@ -19,14 +19,14 @@ namespace webrtc { class MockAgc : public Agc { public: virtual ~MockAgc() {} - MOCK_METHOD3(Process, - void(const int16_t* audio, size_t length, int sample_rate_hz)); - MOCK_METHOD1(GetRmsErrorDb, bool(int* error)); - MOCK_METHOD0(Reset, void()); - MOCK_METHOD1(set_target_level_dbfs, int(int level)); - MOCK_CONST_METHOD0(target_level_dbfs, int()); - MOCK_METHOD1(EnableStandaloneVad, void(bool enable)); - MOCK_CONST_METHOD0(standalone_vad_enabled, bool()); + MOCK_METHOD(void, + Process, + (const int16_t* audio, size_t length, int sample_rate_hz), + (override)); + MOCK_METHOD(bool, GetRmsErrorDb, (int* error), (override)); + MOCK_METHOD(void, Reset, (), (override)); + MOCK_METHOD(int, set_target_level_dbfs, (int level), (override)); + MOCK_METHOD(int, target_level_dbfs, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_processing/agc2/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/agc2/BUILD.gn index 8d9bb147311..bfef2252c3e 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/agc2/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/agc2/BUILD.gn @@ -153,6 +153,7 @@ rtc_library("noise_level_estimator") { "../../../common_audio/third_party/ooura:fft_size_128", "../../../rtc_base:checks", "../../../rtc_base:macromagic", + "../../../system_wrappers:cpu_features_api", ] configs += [ "..:apm_debug_dump" ] diff --git a/chromium/third_party/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn index 5d96fad5834..99b4e82488c 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn @@ -117,9 +117,9 @@ if (rtc_include_tests) { "../../../../rtc_base/system:arch", "../../../../test:test_support", "../../utility:pffft_wrapper", - "//third_party/abseil-cpp/absl/memory", "//third_party/rnnoise:rnn_vad", ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] data = unittest_resources if (is_ios) { deps += [ ":unittests_bundle_data" ] diff --git a/chromium/third_party/webrtc/modules/audio_processing/agc2/signal_classifier.cc b/chromium/third_party/webrtc/modules/audio_processing/agc2/signal_classifier.cc index 8778c494265..38334f7ec5b 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/agc2/signal_classifier.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/agc2/signal_classifier.cc @@ -19,10 +19,19 @@ #include "modules/audio_processing/agc2/noise_spectrum_estimator.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/checks.h" +#include "system_wrappers/include/cpu_features_wrapper.h" namespace webrtc { namespace { +bool IsSse2Available() { +#if defined(WEBRTC_ARCH_X86_FAMILY) + return WebRtc_GetCPUInfo(kSSE2) != 0; +#else + return false; +#endif +} + void RemoveDcLevel(rtc::ArrayView<float> x) { RTC_DCHECK_LT(0, x.size()); float mean = std::accumulate(x.data(), x.data() + x.size(), 0.f); @@ -109,7 +118,8 @@ void SignalClassifier::FrameExtender::ExtendFrame( SignalClassifier::SignalClassifier(ApmDataDumper* data_dumper) : data_dumper_(data_dumper), down_sampler_(data_dumper_), - noise_spectrum_estimator_(data_dumper_) { + noise_spectrum_estimator_(data_dumper_), + ooura_fft_(IsSse2Available()) { Initialize(48000); } SignalClassifier::~SignalClassifier() {} diff --git a/chromium/third_party/webrtc/modules/audio_processing/audio_buffer_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/audio_buffer_unittest.cc index 7cb51ca5f1a..f3b2ddc6895 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/audio_buffer_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/audio_buffer_unittest.cc @@ -40,7 +40,7 @@ TEST(AudioBufferTest, SetNumChannelsSetsChannelBuffersNumChannels) { } #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST(AudioBufferTest, SetNumChannelsDeathTest) { +TEST(AudioBufferDeathTest, SetNumChannelsDeathTest) { AudioBuffer ab(kSampleRateHz, kMono, kSampleRateHz, kMono, kSampleRateHz, kMono); RTC_EXPECT_DEATH(ab.set_num_channels(kStereo), "num_channels"); diff --git a/chromium/third_party/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc index 71352bc65ab..8f28941cdfc 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/audio_processing_impl_unittest.cc @@ -37,13 +37,13 @@ class MockInitialize : public AudioProcessingImpl { explicit MockInitialize(const webrtc::Config& config) : AudioProcessingImpl(config) {} - MOCK_METHOD0(InitializeLocked, int()); + MOCK_METHOD(int, InitializeLocked, (), (override)); int RealInitializeLocked() RTC_NO_THREAD_SAFETY_ANALYSIS { return AudioProcessingImpl::InitializeLocked(); } - MOCK_CONST_METHOD0(AddRef, void()); - MOCK_CONST_METHOD0(Release, rtc::RefCountReleaseStatus()); + MOCK_METHOD(void, AddRef, (), (const, override)); + MOCK_METHOD(rtc::RefCountReleaseStatus, Release, (), (const, override)); }; // Creates MockEchoControl instances and provides a raw pointer access to diff --git a/chromium/third_party/webrtc/modules/audio_processing/audio_processing_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/audio_processing_unittest.cc index 90413a84bee..93ddc973660 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/audio_processing_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/audio_processing_unittest.cc @@ -962,49 +962,51 @@ TEST_F(ApmTest, GainControl) { } #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST_F(ApmTest, GainControlDiesOnTooLowTargetLevelDbfs) { +using ApmDeathTest = ApmTest; + +TEST_F(ApmDeathTest, GainControlDiesOnTooLowTargetLevelDbfs) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.target_level_dbfs = -1; EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, GainControlDiesOnTooHighTargetLevelDbfs) { +TEST_F(ApmDeathTest, GainControlDiesOnTooHighTargetLevelDbfs) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.target_level_dbfs = 32; EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, GainControlDiesOnTooLowCompressionGainDb) { +TEST_F(ApmDeathTest, GainControlDiesOnTooLowCompressionGainDb) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.compression_gain_db = -1; EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, GainControlDiesOnTooHighCompressionGainDb) { +TEST_F(ApmDeathTest, GainControlDiesOnTooHighCompressionGainDb) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.compression_gain_db = 91; EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, GainControlDiesOnTooLowAnalogLevelLowerLimit) { +TEST_F(ApmDeathTest, GainControlDiesOnTooLowAnalogLevelLowerLimit) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.analog_level_minimum = -1; EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, GainControlDiesOnTooHighAnalogLevelUpperLimit) { +TEST_F(ApmDeathTest, GainControlDiesOnTooHighAnalogLevelUpperLimit) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.analog_level_maximum = 65536; EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, GainControlDiesOnInvertedAnalogLevelLimits) { +TEST_F(ApmDeathTest, GainControlDiesOnInvertedAnalogLevelLimits) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.analog_level_minimum = 512; @@ -1012,7 +1014,7 @@ TEST_F(ApmTest, GainControlDiesOnInvertedAnalogLevelLimits) { EXPECT_DEATH(apm_->ApplyConfig(config), ""); } -TEST_F(ApmTest, ApmDiesOnTooLowAnalogLevel) { +TEST_F(ApmDeathTest, ApmDiesOnTooLowAnalogLevel) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.analog_level_minimum = 255; @@ -1021,7 +1023,7 @@ TEST_F(ApmTest, ApmDiesOnTooLowAnalogLevel) { EXPECT_DEATH(apm_->set_stream_analog_level(254), ""); } -TEST_F(ApmTest, ApmDiesOnTooHighAnalogLevel) { +TEST_F(ApmDeathTest, ApmDiesOnTooHighAnalogLevel) { auto config = apm_->GetConfig(); config.gain_controller1.enabled = true; config.gain_controller1.analog_level_minimum = 255; @@ -2414,7 +2416,7 @@ TEST(RuntimeSettingTest, TestDefaultCtor) { EXPECT_EQ(AudioProcessing::RuntimeSetting::Type::kNotSpecified, s.type()); } -TEST(RuntimeSettingTest, TestCapturePreGain) { +TEST(RuntimeSettingDeathTest, TestCapturePreGain) { using Type = AudioProcessing::RuntimeSetting::Type; { auto s = AudioProcessing::RuntimeSetting::CreateCapturePreGain(1.25f); @@ -2429,7 +2431,7 @@ TEST(RuntimeSettingTest, TestCapturePreGain) { #endif } -TEST(RuntimeSettingTest, TestCaptureFixedPostGain) { +TEST(RuntimeSettingDeathTest, TestCaptureFixedPostGain) { using Type = AudioProcessing::RuntimeSetting::Type; { auto s = AudioProcessing::RuntimeSetting::CreateCaptureFixedPostGain(1.25f); diff --git a/chromium/third_party/webrtc/modules/audio_processing/include/mock_audio_processing.h b/chromium/third_party/webrtc/modules/audio_processing/include/mock_audio_processing.h index bdae99a91a2..562b23f7d5d 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/include/mock_audio_processing.h +++ b/chromium/third_party/webrtc/modules/audio_processing/include/mock_audio_processing.h @@ -24,35 +24,47 @@ namespace test { class MockCustomProcessing : public CustomProcessing { public: virtual ~MockCustomProcessing() {} - MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels)); - MOCK_METHOD1(Process, void(AudioBuffer* audio)); - MOCK_METHOD1(SetRuntimeSetting, - void(AudioProcessing::RuntimeSetting setting)); - MOCK_CONST_METHOD0(ToString, std::string()); + MOCK_METHOD(void, + Initialize, + (int sample_rate_hz, int num_channels), + (override)); + MOCK_METHOD(void, Process, (AudioBuffer * audio), (override)); + MOCK_METHOD(void, + SetRuntimeSetting, + (AudioProcessing::RuntimeSetting setting), + (override)); + MOCK_METHOD(std::string, ToString, (), (const, override)); }; class MockCustomAudioAnalyzer : public CustomAudioAnalyzer { public: virtual ~MockCustomAudioAnalyzer() {} - MOCK_METHOD2(Initialize, void(int sample_rate_hz, int num_channels)); - MOCK_METHOD1(Analyze, void(const AudioBuffer* audio)); - MOCK_CONST_METHOD0(ToString, std::string()); + MOCK_METHOD(void, + Initialize, + (int sample_rate_hz, int num_channels), + (override)); + MOCK_METHOD(void, Analyze, (const AudioBuffer* audio), (override)); + MOCK_METHOD(std::string, ToString, (), (const, override)); }; class MockEchoControl : public EchoControl { public: virtual ~MockEchoControl() {} - MOCK_METHOD1(AnalyzeRender, void(AudioBuffer* render)); - MOCK_METHOD1(AnalyzeCapture, void(AudioBuffer* capture)); - MOCK_METHOD2(ProcessCapture, - void(AudioBuffer* capture, bool echo_path_change)); - MOCK_METHOD3(ProcessCapture, - void(AudioBuffer* capture, - AudioBuffer* linear_output, - bool echo_path_change)); - MOCK_CONST_METHOD0(GetMetrics, Metrics()); - MOCK_METHOD1(SetAudioBufferDelay, void(int delay_ms)); - MOCK_CONST_METHOD0(ActiveProcessing, bool()); + MOCK_METHOD(void, AnalyzeRender, (AudioBuffer * render), (override)); + MOCK_METHOD(void, AnalyzeCapture, (AudioBuffer * capture), (override)); + MOCK_METHOD(void, + ProcessCapture, + (AudioBuffer * capture, bool echo_path_change), + (override)); + MOCK_METHOD(void, + ProcessCapture, + (AudioBuffer * capture, + AudioBuffer* linear_output, + bool echo_path_change), + (override)); + MOCK_METHOD(Metrics, GetMetrics, (), (const, override)); + MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, ActiveProcessing, (), (const, override)); }; class MockAudioProcessing : public ::testing::NiceMock<AudioProcessing> { @@ -61,87 +73,93 @@ class MockAudioProcessing : public ::testing::NiceMock<AudioProcessing> { virtual ~MockAudioProcessing() {} - MOCK_METHOD0(Initialize, int()); - MOCK_METHOD6(Initialize, - int(int capture_input_sample_rate_hz, - int capture_output_sample_rate_hz, - int render_sample_rate_hz, - ChannelLayout capture_input_layout, - ChannelLayout capture_output_layout, - ChannelLayout render_input_layout)); - MOCK_METHOD1(Initialize, int(const ProcessingConfig& processing_config)); - MOCK_METHOD1(ApplyConfig, void(const Config& config)); - MOCK_METHOD1(SetExtraOptions, void(const webrtc::Config& config)); - MOCK_CONST_METHOD0(proc_sample_rate_hz, int()); - MOCK_CONST_METHOD0(proc_split_sample_rate_hz, int()); - MOCK_CONST_METHOD0(num_input_channels, size_t()); - MOCK_CONST_METHOD0(num_proc_channels, size_t()); - MOCK_CONST_METHOD0(num_output_channels, size_t()); - MOCK_CONST_METHOD0(num_reverse_channels, size_t()); - MOCK_METHOD1(set_output_will_be_muted, void(bool muted)); - MOCK_METHOD1(SetRuntimeSetting, void(RuntimeSetting setting)); - MOCK_METHOD4(ProcessStream, - int(const int16_t* const src, - const StreamConfig& input_config, - const StreamConfig& output_config, - int16_t* const dest)); - MOCK_METHOD7(ProcessStream, - int(const float* const* src, - size_t samples_per_channel, - int input_sample_rate_hz, - ChannelLayout input_layout, - int output_sample_rate_hz, - ChannelLayout output_layout, - float* const* dest)); - MOCK_METHOD4(ProcessStream, - int(const float* const* src, - const StreamConfig& input_config, - const StreamConfig& output_config, - float* const* dest)); - MOCK_METHOD4(ProcessReverseStream, - int(const int16_t* const src, - const StreamConfig& input_config, - const StreamConfig& output_config, - int16_t* const dest)); - MOCK_METHOD4(AnalyzeReverseStream, - int(const float* const* data, - size_t samples_per_channel, - int sample_rate_hz, - ChannelLayout layout)); - MOCK_METHOD2(AnalyzeReverseStream, - int(const float* const* data, - const StreamConfig& reverse_config)); - MOCK_METHOD4(ProcessReverseStream, - int(const float* const* src, - const StreamConfig& input_config, - const StreamConfig& output_config, - float* const* dest)); - MOCK_CONST_METHOD1( - GetLinearAecOutput, - bool(rtc::ArrayView<std::array<float, 160>> linear_output)); - MOCK_METHOD1(set_stream_delay_ms, int(int delay)); - MOCK_CONST_METHOD0(stream_delay_ms, int()); - MOCK_CONST_METHOD0(was_stream_delay_set, bool()); - MOCK_METHOD1(set_stream_key_pressed, void(bool key_pressed)); - MOCK_METHOD1(set_delay_offset_ms, void(int offset)); - MOCK_CONST_METHOD0(delay_offset_ms, int()); - MOCK_METHOD1(set_stream_analog_level, void(int)); - MOCK_CONST_METHOD0(recommended_stream_analog_level, int()); - MOCK_METHOD3(CreateAndAttachAecDump, - bool(const std::string& file_name, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue)); - MOCK_METHOD3(CreateAndAttachAecDump, - bool(FILE* handle, - int64_t max_log_size_bytes, - rtc::TaskQueue* worker_queue)); - MOCK_METHOD1(AttachAecDump, void(std::unique_ptr<AecDump>)); - MOCK_METHOD0(DetachAecDump, void()); + MOCK_METHOD(int, Initialize, (), (override)); + MOCK_METHOD(int, + Initialize, + (int capture_input_sample_rate_hz, + int capture_output_sample_rate_hz, + int render_sample_rate_hz, + ChannelLayout capture_input_layout, + ChannelLayout capture_output_layout, + ChannelLayout render_input_layout), + (override)); + MOCK_METHOD(int, + Initialize, + (const ProcessingConfig& processing_config), + (override)); + MOCK_METHOD(void, ApplyConfig, (const Config& config), (override)); + MOCK_METHOD(void, + SetExtraOptions, + (const webrtc::Config& config), + (override)); + MOCK_METHOD(int, proc_sample_rate_hz, (), (const, override)); + MOCK_METHOD(int, proc_split_sample_rate_hz, (), (const, override)); + MOCK_METHOD(size_t, num_input_channels, (), (const, override)); + MOCK_METHOD(size_t, num_proc_channels, (), (const, override)); + MOCK_METHOD(size_t, num_output_channels, (), (const, override)); + MOCK_METHOD(size_t, num_reverse_channels, (), (const, override)); + MOCK_METHOD(void, set_output_will_be_muted, (bool muted), (override)); + MOCK_METHOD(void, SetRuntimeSetting, (RuntimeSetting setting), (override)); + MOCK_METHOD(int, + ProcessStream, + (const int16_t* const src, + const StreamConfig& input_config, + const StreamConfig& output_config, + int16_t* const dest), + (override)); + MOCK_METHOD(int, + ProcessStream, + (const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest), + (override)); + MOCK_METHOD(int, + ProcessReverseStream, + (const int16_t* const src, + const StreamConfig& input_config, + const StreamConfig& output_config, + int16_t* const dest), + (override)); + MOCK_METHOD(int, + AnalyzeReverseStream, + (const float* const* data, const StreamConfig& reverse_config), + (override)); + MOCK_METHOD(int, + ProcessReverseStream, + (const float* const* src, + const StreamConfig& input_config, + const StreamConfig& output_config, + float* const* dest), + (override)); + MOCK_METHOD(bool, + GetLinearAecOutput, + ((rtc::ArrayView<std::array<float, 160>> linear_output)), + (const, override)); + MOCK_METHOD(int, set_stream_delay_ms, (int delay), (override)); + MOCK_METHOD(int, stream_delay_ms, (), (const, override)); + MOCK_METHOD(void, set_stream_key_pressed, (bool key_pressed), (override)); + MOCK_METHOD(void, set_stream_analog_level, (int), (override)); + MOCK_METHOD(int, recommended_stream_analog_level, (), (const, override)); + MOCK_METHOD(bool, + CreateAndAttachAecDump, + (const std::string& file_name, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue), + (override)); + MOCK_METHOD(bool, + CreateAndAttachAecDump, + (FILE * handle, + int64_t max_log_size_bytes, + rtc::TaskQueue* worker_queue), + (override)); + MOCK_METHOD(void, AttachAecDump, (std::unique_ptr<AecDump>), (override)); + MOCK_METHOD(void, DetachAecDump, (), (override)); - MOCK_METHOD0(GetStatistics, AudioProcessingStats()); - MOCK_METHOD1(GetStatistics, AudioProcessingStats(bool)); + MOCK_METHOD(AudioProcessingStats, GetStatistics, (), (override)); + MOCK_METHOD(AudioProcessingStats, GetStatistics, (bool), (override)); - MOCK_CONST_METHOD0(GetConfig, AudioProcessing::Config()); + MOCK_METHOD(AudioProcessing::Config, GetConfig, (), (const, override)); }; } // namespace test diff --git a/chromium/third_party/webrtc/modules/audio_processing/ns/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/ns/BUILD.gn index 442a313e640..7197705c3d0 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/ns/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/ns/BUILD.gn @@ -64,8 +64,8 @@ rtc_static_library("ns") { "../../../system_wrappers:field_trial", "../../../system_wrappers:metrics", "../utility:cascaded_biquad_filter", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } if (rtc_include_tests) { @@ -90,8 +90,8 @@ if (rtc_include_tests) { "../../../system_wrappers:cpu_features_api", "../../../test:test_support", "../utility:cascaded_biquad_filter", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] defines = [] diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc b/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc index b3b113da448..f5bd6452e30 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.cc @@ -66,8 +66,11 @@ bool VerifyFloatBitExactness(const webrtc::audioproc::Stream& msg, AecDumpBasedSimulator::AecDumpBasedSimulator( const SimulationSettings& settings, + rtc::scoped_refptr<AudioProcessing> audio_processing, std::unique_ptr<AudioProcessingBuilder> ap_builder) - : AudioProcessingSimulator(settings, std::move(ap_builder)) { + : AudioProcessingSimulator(settings, + std::move(audio_processing), + std::move(ap_builder)) { MaybeOpenCallOrderFile(); } @@ -206,7 +209,8 @@ void AecDumpBasedSimulator::PrepareReverseProcessStreamCall( } void AecDumpBasedSimulator::Process() { - CreateAudioProcessor(); + ConfigureAudioProcessor(); + if (settings_.artificial_nearend_filename) { std::unique_ptr<WavReader> artificial_nearend_file( new WavReader(settings_.artificial_nearend_filename->c_str())); @@ -237,7 +241,7 @@ void AecDumpBasedSimulator::Process() { fclose(dump_input_file_); } - DestroyAudioProcessor(); + DetachAecDump(); } void AecDumpBasedSimulator::HandleEvent( diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h b/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h index ef032d03160..092b82bdbc6 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/aec_dump_based_simulator.h @@ -33,6 +33,7 @@ namespace test { class AecDumpBasedSimulator final : public AudioProcessingSimulator { public: AecDumpBasedSimulator(const SimulationSettings& settings, + rtc::scoped_refptr<AudioProcessing> audio_processing, std::unique_ptr<AudioProcessingBuilder> ap_builder); ~AecDumpBasedSimulator() override; diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.cc b/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.cc index a37a83f1e3c..adbc298e9eb 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.cc @@ -113,10 +113,10 @@ SimulationSettings::~SimulationSettings() = default; AudioProcessingSimulator::AudioProcessingSimulator( const SimulationSettings& settings, + rtc::scoped_refptr<AudioProcessing> audio_processing, std::unique_ptr<AudioProcessingBuilder> ap_builder) : settings_(settings), - ap_builder_(ap_builder ? std::move(ap_builder) - : std::make_unique<AudioProcessingBuilder>()), + ap_(std::move(audio_processing)), analog_mic_level_(settings.initial_mic_level), fake_recording_device_( settings.initial_mic_level, @@ -139,6 +139,51 @@ AudioProcessingSimulator::AudioProcessingSimulator( if (settings_.simulate_mic_gain) RTC_LOG(LS_VERBOSE) << "Simulating analog mic gain"; + + // Create the audio processing object. + RTC_CHECK(!(ap_ && ap_builder)) + << "The AudioProcessing and the AudioProcessingBuilder cannot both be " + "specified at the same time."; + + if (ap_) { + RTC_CHECK(!settings_.aec_settings_filename); + RTC_CHECK(!settings_.print_aec_parameter_values); + } else { + // Use specied builder if such is provided, otherwise create a new builder. + std::unique_ptr<AudioProcessingBuilder> builder = + !!ap_builder ? std::move(ap_builder) + : std::make_unique<AudioProcessingBuilder>(); + + // Create and set an EchoCanceller3Factory if needed. + const bool use_aec = settings_.use_aec && *settings_.use_aec; + if (use_aec) { + EchoCanceller3Config cfg; + if (settings_.aec_settings_filename) { + if (settings_.use_verbose_logging) { + std::cout << "Reading AEC Parameters from JSON input." << std::endl; + } + cfg = ReadAec3ConfigFromJsonFile(*settings_.aec_settings_filename); + } + + if (settings_.linear_aec_output_filename) { + cfg.filter.export_linear_aec_output = true; + } + + if (settings_.print_aec_parameter_values) { + if (!settings_.use_quiet_output) { + std::cout << "AEC settings:" << std::endl; + } + std::cout << Aec3ConfigToJsonString(cfg) << std::endl; + } + + auto echo_control_factory = std::make_unique<EchoCanceller3Factory>(cfg); + builder->SetEchoControlFactory(std::move(echo_control_factory)); + } + + // Create an audio processing object. + ap_ = builder->Create(); + RTC_CHECK(ap_); + } } AudioProcessingSimulator::~AudioProcessingSimulator() { @@ -369,16 +414,14 @@ void AudioProcessingSimulator::SetupOutput() { ++output_reset_counter_; } -void AudioProcessingSimulator::DestroyAudioProcessor() { +void AudioProcessingSimulator::DetachAecDump() { if (settings_.aec_dump_output_filename) { ap_->DetachAecDump(); } } -void AudioProcessingSimulator::CreateAudioProcessor() { - Config config; +void AudioProcessingSimulator::ConfigureAudioProcessor() { AudioProcessing::Config apm_config; - std::unique_ptr<EchoControlFactory> echo_control_factory; if (settings_.use_ts) { apm_config.transient_suppression.enabled = *settings_.use_ts; } @@ -421,29 +464,6 @@ void AudioProcessingSimulator::CreateAudioProcessor() { apm_config.echo_canceller.export_linear_aec_output = !!settings_.linear_aec_output_filename; - if (use_aec) { - EchoCanceller3Config cfg; - if (settings_.aec_settings_filename) { - if (settings_.use_verbose_logging) { - std::cout << "Reading AEC Parameters from JSON input." << std::endl; - } - cfg = ReadAec3ConfigFromJsonFile(*settings_.aec_settings_filename); - } - - if (settings_.linear_aec_output_filename) { - cfg.filter.export_linear_aec_output = true; - } - - echo_control_factory.reset(new EchoCanceller3Factory(cfg)); - - if (settings_.print_aec_parameter_values) { - if (!settings_.use_quiet_output) { - std::cout << "AEC settings:" << std::endl; - } - std::cout << Aec3ConfigToJsonString(cfg) << std::endl; - } - } - if (settings_.use_hpf) { apm_config.high_pass_filter.enabled = *settings_.use_hpf; } @@ -512,14 +532,6 @@ void AudioProcessingSimulator::CreateAudioProcessor() { *settings_.ns_analysis_on_linear_aec_output; } - RTC_CHECK(ap_builder_); - if (echo_control_factory) { - ap_builder_->SetEchoControlFactory(std::move(echo_control_factory)); - } - ap_.reset((*ap_builder_).Create(config)); - - RTC_CHECK(ap_); - ap_->ApplyConfig(apm_config); if (settings_.use_ts) { diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.h b/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.h index fa6efc2842c..8579f4b4d0c 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/audio_processing_simulator.h @@ -150,8 +150,8 @@ struct SimulationSettings { // Provides common functionality for performing audioprocessing simulations. class AudioProcessingSimulator { public: - AudioProcessingSimulator(const SimulationSettings& settings, + rtc::scoped_refptr<AudioProcessing> audio_processing, std::unique_ptr<AudioProcessingBuilder> ap_builder); virtual ~AudioProcessingSimulator(); @@ -174,8 +174,8 @@ class AudioProcessingSimulator { protected: void ProcessStream(bool fixed_interface); void ProcessReverseStream(bool fixed_interface); - void CreateAudioProcessor(); - void DestroyAudioProcessor(); + void ConfigureAudioProcessor(); + void DetachAecDump(); void SetupBuffersConfigsOutputs(int input_sample_rate_hz, int output_sample_rate_hz, int reverse_input_sample_rate_hz, @@ -186,8 +186,7 @@ class AudioProcessingSimulator { int reverse_output_num_channels); const SimulationSettings settings_; - std::unique_ptr<AudioProcessing> ap_; - std::unique_ptr<AudioProcessingBuilder> ap_builder_; + rtc::scoped_refptr<AudioProcessing> ap_; std::unique_ptr<ChannelBuffer<float>> in_buf_; std::unique_ptr<ChannelBuffer<float>> out_buf_; diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.cc b/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.cc index d9a4227eb72..ab395f1018c 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.cc @@ -457,7 +457,10 @@ void ReportConditionalErrorAndExit(bool condition, const std::string& message) { } } -void PerformBasicParameterSanityChecks(const SimulationSettings& settings) { +void PerformBasicParameterSanityChecks( + const SimulationSettings& settings, + bool pre_constructed_ap_provided, + bool pre_constructed_ap_builder_provided) { if (settings.input_filename || settings.reverse_input_filename) { ReportConditionalErrorAndExit( !!settings.aec_dump_input_filename, @@ -624,21 +627,41 @@ void PerformBasicParameterSanityChecks(const SimulationSettings& settings) { settings.pre_amplifier_gain_factor.has_value(), "Error: --pre_amplifier_gain_factor needs --pre_amplifier to be " "specified and set.\n"); -} -} // namespace + ReportConditionalErrorAndExit( + pre_constructed_ap_provided && pre_constructed_ap_builder_provided, + "Error: The AudioProcessing and the AudioProcessingBuilder cannot both " + "be specified at the same time.\n"); -int AudioprocFloatImpl(std::unique_ptr<AudioProcessingBuilder> ap_builder, - int argc, - char* argv[], - absl::string_view input_aecdump, - std::vector<float>* processed_capture_samples) { + ReportConditionalErrorAndExit( + settings.aec_settings_filename && pre_constructed_ap_provided, + "Error: The aec_settings_filename cannot be specified when a " + "pre-constructed audio processing object is provided.\n"); + + ReportConditionalErrorAndExit( + settings.aec_settings_filename && pre_constructed_ap_provided, + "Error: The print_aec_parameter_values cannot be set when a " + "pre-constructed audio processing object is provided.\n"); + + if (settings.linear_aec_output_filename && pre_constructed_ap_provided) { + std::cout << "Warning: For the linear AEC output to be stored, this must " + "be configured in the AEC that is part of the provided " + "AudioProcessing object." + << std::endl; + } +} + +int RunSimulation(rtc::scoped_refptr<AudioProcessing> audio_processing, + std::unique_ptr<AudioProcessingBuilder> ap_builder, + int argc, + char* argv[], + absl::string_view input_aecdump, + std::vector<float>* processed_capture_samples) { std::vector<char*> args = absl::ParseCommandLine(argc, argv); if (args.size() != 1) { printf("%s", kUsageDescription); return 1; } - // InitFieldTrialsFromString stores the char*, so the char array must // outlive the application. const std::string field_trials = absl::GetFlag(FLAGS_force_fieldtrials); @@ -650,13 +673,15 @@ int AudioprocFloatImpl(std::unique_ptr<AudioProcessingBuilder> ap_builder, settings.processed_capture_samples = processed_capture_samples; RTC_CHECK(settings.processed_capture_samples); } - PerformBasicParameterSanityChecks(settings); + PerformBasicParameterSanityChecks(settings, !!audio_processing, !!ap_builder); std::unique_ptr<AudioProcessingSimulator> processor; if (settings.aec_dump_input_filename || settings.aec_dump_input_string) { - processor.reset(new AecDumpBasedSimulator(settings, std::move(ap_builder))); + processor.reset(new AecDumpBasedSimulator( + settings, std::move(audio_processing), std::move(ap_builder))); } else { - processor.reset(new WavBasedSimulator(settings, std::move(ap_builder))); + processor.reset(new WavBasedSimulator(settings, std::move(audio_processing), + std::move(ap_builder))); } processor->Process(); @@ -680,5 +705,24 @@ int AudioprocFloatImpl(std::unique_ptr<AudioProcessingBuilder> ap_builder, return 0; } +} // namespace + +int AudioprocFloatImpl(rtc::scoped_refptr<AudioProcessing> audio_processing, + int argc, + char* argv[]) { + return RunSimulation( + std::move(audio_processing), /*ap_builder=*/nullptr, argc, argv, + /*input_aecdump=*/"", /*processed_capture_samples=*/nullptr); +} + +int AudioprocFloatImpl(std::unique_ptr<AudioProcessingBuilder> ap_builder, + int argc, + char* argv[], + absl::string_view input_aecdump, + std::vector<float>* processed_capture_samples) { + return RunSimulation(/*audio_processing=*/nullptr, std::move(ap_builder), + argc, argv, input_aecdump, processed_capture_samples); +} + } // namespace test } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.h b/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.h index 9a9013c6445..0687c43a5d6 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/audioproc_float_impl.h @@ -24,6 +24,21 @@ namespace test { // via the |argv| argument. Pass |processed_capture_samples| to write in it the // samples processed on the capture side; if |processed_capture_samples| is not // passed, the output file can optionally be specified via the |argv| argument. +// Any audio_processing object specified in the input is used for the +// simulation. Note that when the audio_processing object is specified all +// functionality that relies on using the internal builder is deactivated, +// since the AudioProcessing object is already created and the builder is not +// used in the simulation. +int AudioprocFloatImpl(rtc::scoped_refptr<AudioProcessing> audio_processing, + int argc, + char* argv[]); + +// This function implements the audio processing simulation utility. Pass +// |input_aecdump| to provide the content of an AEC dump file as a string; if +// |input_aecdump| is not passed, a WAV or AEC input dump file must be specified +// via the |argv| argument. Pass |processed_capture_samples| to write in it the +// samples processed on the capture side; if |processed_capture_samples| is not +// passed, the output file can optionally be specified via the |argv| argument. int AudioprocFloatImpl(std::unique_ptr<AudioProcessingBuilder> ap_builder, int argc, char* argv[], diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn b/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn index fb532befb95..b311abdbd17 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn +++ b/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/BUILD.gn @@ -68,6 +68,6 @@ rtc_library("unittest") { "../../../../test:fileutils", "../../../../test:test_support", "//testing/gtest", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h b/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h index 591299eefe5..94e20b9ec60 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader.h @@ -28,12 +28,12 @@ class MockWavReader : public WavReaderInterface { ~MockWavReader(); // TODO(alessiob): use ON_CALL to return random samples if needed. - MOCK_METHOD1(ReadFloatSamples, size_t(rtc::ArrayView<float>)); - MOCK_METHOD1(ReadInt16Samples, size_t(rtc::ArrayView<int16_t>)); + MOCK_METHOD(size_t, ReadFloatSamples, (rtc::ArrayView<float>), (override)); + MOCK_METHOD(size_t, ReadInt16Samples, (rtc::ArrayView<int16_t>), (override)); - MOCK_CONST_METHOD0(SampleRate, int()); - MOCK_CONST_METHOD0(NumChannels, size_t()); - MOCK_CONST_METHOD0(NumSamples, size_t()); + MOCK_METHOD(int, SampleRate, (), (const, override)); + MOCK_METHOD(size_t, NumChannels, (), (const, override)); + MOCK_METHOD(size_t, NumSamples, (), (const, override)); private: const int sample_rate_; diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h b/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h index e84f5f35ccb..c2db85f6f68 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/conversational_speech/mock_wavreader_factory.h @@ -36,8 +36,10 @@ class MockWavReaderFactory : public WavReaderAbstractFactory { explicit MockWavReaderFactory(const Params& default_params); ~MockWavReaderFactory(); - MOCK_CONST_METHOD1(Create, - std::unique_ptr<WavReaderInterface>(const std::string&)); + MOCK_METHOD(std::unique_ptr<WavReaderInterface>, + Create, + (const std::string&), + (const, override)); private: // Creates a MockWavReader instance using the parameters in diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/echo_control_mock.h b/chromium/third_party/webrtc/modules/audio_processing/test/echo_control_mock.h index 95d3be5cdfa..927de43ae09 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/echo_control_mock.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/echo_control_mock.h @@ -20,17 +20,21 @@ class AudioBuffer; class MockEchoControl : public EchoControl { public: - MOCK_METHOD1(AnalyzeRender, void(AudioBuffer* render)); - MOCK_METHOD1(AnalyzeCapture, void(AudioBuffer* capture)); - MOCK_METHOD2(ProcessCapture, - void(AudioBuffer* capture, bool echo_path_change)); - MOCK_METHOD3(ProcessCapture, - void(AudioBuffer* capture, - AudioBuffer* linear_output, - bool echo_path_change)); - MOCK_CONST_METHOD0(GetMetrics, EchoControl::Metrics()); - MOCK_METHOD1(SetAudioBufferDelay, void(int delay_ms)); - MOCK_CONST_METHOD0(ActiveProcessing, bool()); + MOCK_METHOD(void, AnalyzeRender, (AudioBuffer * render), (override)); + MOCK_METHOD(void, AnalyzeCapture, (AudioBuffer * capture), (override)); + MOCK_METHOD(void, + ProcessCapture, + (AudioBuffer * capture, bool echo_path_change), + (override)); + MOCK_METHOD(void, + ProcessCapture, + (AudioBuffer * capture, + AudioBuffer* linear_output, + bool echo_path_change), + (override)); + MOCK_METHOD(EchoControl::Metrics, GetMetrics, (), (const, override)); + MOCK_METHOD(void, SetAudioBufferDelay, (int delay_ms), (override)); + MOCK_METHOD(bool, ActiveProcessing, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.cc b/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.cc index 7179fc3431f..75946fb3fa6 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.cc @@ -56,8 +56,18 @@ WavBasedSimulator::GetCustomEventChain(const std::string& filename) { WavBasedSimulator::WavBasedSimulator( const SimulationSettings& settings, + rtc::scoped_refptr<AudioProcessing> audio_processing, std::unique_ptr<AudioProcessingBuilder> ap_builder) - : AudioProcessingSimulator(settings, std::move(ap_builder)) {} + : AudioProcessingSimulator(settings, + std::move(audio_processing), + std::move(ap_builder)) { + if (settings_.call_order_input_filename) { + call_chain_ = WavBasedSimulator::GetCustomEventChain( + *settings_.call_order_input_filename); + } else { + call_chain_ = WavBasedSimulator::GetDefaultEventChain(); + } +} WavBasedSimulator::~WavBasedSimulator() = default; @@ -89,13 +99,7 @@ void WavBasedSimulator::PrepareReverseProcessStreamCall() { } void WavBasedSimulator::Process() { - if (settings_.call_order_input_filename) { - call_chain_ = WavBasedSimulator::GetCustomEventChain( - *settings_.call_order_input_filename); - } else { - call_chain_ = WavBasedSimulator::GetDefaultEventChain(); - } - CreateAudioProcessor(); + ConfigureAudioProcessor(); Initialize(); @@ -120,7 +124,7 @@ void WavBasedSimulator::Process() { call_chain_index = (call_chain_index + 1) % call_chain_.size(); } - DestroyAudioProcessor(); + DetachAecDump(); } bool WavBasedSimulator::HandleProcessStreamCall() { diff --git a/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.h b/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.h index 991f1dbaadf..3adbe7022c4 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.h +++ b/chromium/third_party/webrtc/modules/audio_processing/test/wav_based_simulator.h @@ -23,6 +23,7 @@ namespace test { class WavBasedSimulator final : public AudioProcessingSimulator { public: WavBasedSimulator(const SimulationSettings& settings, + rtc::scoped_refptr<AudioProcessing> audio_processing, std::unique_ptr<AudioProcessingBuilder> ap_builder); ~WavBasedSimulator() override; diff --git a/chromium/third_party/webrtc/modules/audio_processing/utility/cascaded_biquad_filter_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/utility/cascaded_biquad_filter_unittest.cc index 989e362a499..ff7022dba41 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/utility/cascaded_biquad_filter_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/utility/cascaded_biquad_filter_unittest.cc @@ -103,7 +103,7 @@ TEST(CascadedBiquadFilter, TransparentConfiguration) { #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) // Verifies that the check of the lengths for the input and output works for the // non-in-place call. -TEST(CascadedBiquadFilter, InputSizeCheckVerification) { +TEST(CascadedBiquadFilterDeathTest, InputSizeCheckVerification) { const std::vector<float> input = CreateInputWithIncreasingValues(10); std::vector<float> output(input.size() - 1); diff --git a/chromium/third_party/webrtc/modules/audio_processing/utility/pffft_wrapper_unittest.cc b/chromium/third_party/webrtc/modules/audio_processing/utility/pffft_wrapper_unittest.cc index 9aed548934e..2ad6849cd47 100644 --- a/chromium/third_party/webrtc/modules/audio_processing/utility/pffft_wrapper_unittest.cc +++ b/chromium/third_party/webrtc/modules/audio_processing/utility/pffft_wrapper_unittest.cc @@ -125,23 +125,24 @@ TEST(PffftTest, CreateWrapperWithValidSize) { #if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -class PffftInvalidSizeTest : public ::testing::Test, - public ::testing::WithParamInterface<size_t> {}; +class PffftInvalidSizeDeathTest : public ::testing::Test, + public ::testing::WithParamInterface<size_t> { +}; -TEST_P(PffftInvalidSizeTest, DoNotCreateRealWrapper) { +TEST_P(PffftInvalidSizeDeathTest, DoNotCreateRealWrapper) { size_t fft_size = GetParam(); ASSERT_FALSE(Pffft::IsValidFftSize(fft_size, Pffft::FftType::kReal)); EXPECT_DEATH(CreatePffftWrapper(fft_size, Pffft::FftType::kReal), ""); } -TEST_P(PffftInvalidSizeTest, DoNotCreateComplexWrapper) { +TEST_P(PffftInvalidSizeDeathTest, DoNotCreateComplexWrapper) { size_t fft_size = GetParam(); ASSERT_FALSE(Pffft::IsValidFftSize(fft_size, Pffft::FftType::kComplex)); EXPECT_DEATH(CreatePffftWrapper(fft_size, Pffft::FftType::kComplex), ""); } INSTANTIATE_TEST_SUITE_P(PffftTest, - PffftInvalidSizeTest, + PffftInvalidSizeDeathTest, ::testing::Values(17, 33, 65, diff --git a/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/BUILD.gn b/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/BUILD.gn index fa95bc186c8..52daad2bce8 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/BUILD.gn +++ b/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/BUILD.gn @@ -51,6 +51,8 @@ rtc_library("goog_cc") { "../../../rtc_base/experiments:rate_control_settings", "../../../system_wrappers", "../../remote_bitrate_estimator", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -64,8 +66,8 @@ rtc_library("link_capacity_estimator") { deps = [ "../../../api/units:data_rate", "../../../rtc_base:safe_minmax", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("pushback_controller") { @@ -79,6 +81,8 @@ rtc_library("pushback_controller") { "../../../api/units:data_size", "../../../rtc_base:checks", "../../../rtc_base/experiments:rate_control_settings", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -100,8 +104,8 @@ rtc_library("alr_detector") { "../../../rtc_base/experiments:alr_experiment", "../../../rtc_base/experiments:field_trial_parser", "../../pacing:interval_budget", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("estimators") { configs += [ ":bwe_test_logging" ] @@ -137,6 +141,8 @@ rtc_library("estimators") { "../../../rtc_base:safe_minmax", "../../../rtc_base/experiments:field_trial_parser", "../../remote_bitrate_estimator", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -163,6 +169,8 @@ rtc_library("loss_based_controller") { "../../../system_wrappers:field_trial", "../../../system_wrappers:metrics", "../../remote_bitrate_estimator", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -189,6 +197,8 @@ rtc_library("delay_based_bwe") { "../../../system_wrappers:metrics", "../../pacing", "../../remote_bitrate_estimator", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -216,6 +226,8 @@ rtc_library("probe_controller") { "../../../rtc_base/experiments:field_trial_parser", "../../../rtc_base/system:unused", "../../../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -240,8 +252,8 @@ if (rtc_include_tests) { "../../../rtc_base:checks", "../../../test/logging:log_writer", "../../remote_bitrate_estimator", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("goog_cc_unittests") { testonly = true diff --git a/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_unittest.cc b/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_unittest.cc index 9031c5d272e..e5b733b1196 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_unittest.cc +++ b/chromium/third_party/webrtc/modules/congestion_controller/goog_cc/acknowledged_bitrate_estimator_unittest.cc @@ -35,10 +35,12 @@ constexpr size_t kPayloadSize = 10; class MockBitrateEstimator : public BitrateEstimator { public: using BitrateEstimator::BitrateEstimator; - MOCK_METHOD3(Update, - void(Timestamp at_time, DataSize data_size, bool in_alr)); - MOCK_CONST_METHOD0(bitrate, absl::optional<DataRate>()); - MOCK_METHOD0(ExpectFastRateChange, void()); + MOCK_METHOD(void, + Update, + (Timestamp at_time, DataSize data_size, bool in_alr), + (override)); + MOCK_METHOD(absl::optional<DataRate>, bitrate, (), (const, override)); + MOCK_METHOD(void, ExpectFastRateChange, (), (override)); }; struct AcknowledgedBitrateEstimatorTestStates { diff --git a/chromium/third_party/webrtc/modules/congestion_controller/pcc/BUILD.gn b/chromium/third_party/webrtc/modules/congestion_controller/pcc/BUILD.gn index d0111725d29..2f378769e7a 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/pcc/BUILD.gn +++ b/chromium/third_party/webrtc/modules/congestion_controller/pcc/BUILD.gn @@ -37,8 +37,8 @@ rtc_library("pcc_controller") { "../../../api/units:timestamp", "../../../rtc_base:checks", "../../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("monitor_interval") { @@ -94,8 +94,8 @@ rtc_library("bitrate_controller") { "../../../api/transport:network_control", "../../../api/units:data_rate", "../../../rtc_base:rtc_base_approved", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } if (rtc_include_tests) { diff --git a/chromium/third_party/webrtc/modules/congestion_controller/pcc/bitrate_controller_unittest.cc b/chromium/third_party/webrtc/modules/congestion_controller/pcc/bitrate_controller_unittest.cc index 6693b7a833c..957d99b1ded 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/pcc/bitrate_controller_unittest.cc +++ b/chromium/third_party/webrtc/modules/congestion_controller/pcc/bitrate_controller_unittest.cc @@ -67,8 +67,10 @@ std::vector<PacketResult> CreatePacketResults( class MockUtilityFunction : public PccUtilityFunctionInterface { public: - MOCK_CONST_METHOD1(Compute, - double(const PccMonitorInterval& monitor_interval)); + MOCK_METHOD(double, + Compute, + (const PccMonitorInterval& monitor_interval), + (const, override)); }; } // namespace diff --git a/chromium/third_party/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc b/chromium/third_party/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc index 95143f71752..b5846237eec 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc +++ b/chromium/third_party/webrtc/modules/congestion_controller/receive_side_congestion_controller_unittest.cc @@ -37,8 +37,10 @@ uint32_t AbsSendTime(int64_t t, int64_t denom) { class MockPacketRouter : public PacketRouter { public: - MOCK_METHOD2(OnReceiveBitrateChanged, - void(const std::vector<uint32_t>& ssrcs, uint32_t bitrate)); + MOCK_METHOD(void, + OnReceiveBitrateChanged, + (const std::vector<uint32_t>& ssrcs, uint32_t bitrate), + (override)); }; const uint32_t kInitialBitrateBps = 60000; diff --git a/chromium/third_party/webrtc/modules/congestion_controller/rtp/BUILD.gn b/chromium/third_party/webrtc/modules/congestion_controller/rtp/BUILD.gn index b444f5495b9..45c53edcb64 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/rtp/BUILD.gn +++ b/chromium/third_party/webrtc/modules/congestion_controller/rtp/BUILD.gn @@ -33,8 +33,8 @@ rtc_library("control_handler") { "../../../rtc_base/synchronization:sequence_checker", "../../../system_wrappers:field_trial", "../../pacing", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] if (!build_with_mozilla) { deps += [ "../../../rtc_base" ] @@ -61,6 +61,8 @@ rtc_library("transport_feedback") { "../../../system_wrappers", "../../../system_wrappers:field_trial", "../../rtp_rtcp:rtp_rtcp_format", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_adapter_unittest.cc b/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_adapter_unittest.cc index 1c74b196d83..3849cb37077 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_adapter_unittest.cc +++ b/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_adapter_unittest.cc @@ -83,8 +83,10 @@ namespace test { class MockStreamFeedbackObserver : public webrtc::StreamFeedbackObserver { public: - MOCK_METHOD1(OnPacketFeedbackVector, - void(std::vector<StreamPacketInfo> packet_feedback_vector)); + MOCK_METHOD(void, + OnPacketFeedbackVector, + (std::vector<StreamPacketInfo> packet_feedback_vector), + (override)); }; class TransportFeedbackAdapterTest : public ::testing::Test { diff --git a/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_demuxer_unittest.cc b/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_demuxer_unittest.cc index dce52de5574..6514a4eda7c 100644 --- a/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_demuxer_unittest.cc +++ b/chromium/third_party/webrtc/modules/congestion_controller/rtp/transport_feedback_demuxer_unittest.cc @@ -21,8 +21,10 @@ static constexpr uint32_t kSsrc = 8492; class MockStreamFeedbackObserver : public webrtc::StreamFeedbackObserver { public: - MOCK_METHOD1(OnPacketFeedbackVector, - void(std::vector<StreamPacketInfo> packet_feedback_vector)); + MOCK_METHOD(void, + OnPacketFeedbackVector, + (std::vector<StreamPacketInfo> packet_feedback_vector), + (override)); }; RtpPacketSendInfo CreatePacket(uint32_t ssrc, diff --git a/chromium/third_party/webrtc/modules/desktop_capture/BUILD.gn b/chromium/third_party/webrtc/modules/desktop_capture/BUILD.gn index 4f93c246fe6..e49e8381f6a 100644 --- a/chromium/third_party/webrtc/modules/desktop_capture/BUILD.gn +++ b/chromium/third_party/webrtc/modules/desktop_capture/BUILD.gn @@ -42,7 +42,8 @@ rtc_library("primitives") { ] if (!build_with_mozilla) { - deps += [ "../../rtc_base" ] # TODO(kjellander): Cleanup in bugs.webrtc.org/3806. + deps += [ "../../rtc_base" ] # TODO(kjellander): Cleanup in + # bugs.webrtc.org/3806. } } @@ -224,7 +225,8 @@ if (is_linux) { rtc_source_set("desktop_capture") { visibility = [ "*" ] - public_deps = [ ":desktop_capture_generic" ] # no-presubmit-check TODO(webrtc:8603) + public_deps = # no-presubmit-check TODO(webrtc:8603) + [ ":desktop_capture_generic" ] if (is_mac) { public_deps += [ ":desktop_capture_objc" ] } @@ -480,6 +482,8 @@ rtc_library("desktop_capture_generic") { "../../system_wrappers", "../../system_wrappers:cpu_features_api", "../../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings", ] diff --git a/chromium/third_party/webrtc/modules/desktop_capture/mock_desktop_capturer_callback.h b/chromium/third_party/webrtc/modules/desktop_capture/mock_desktop_capturer_callback.h index 659239ab9db..6530dc5542a 100644 --- a/chromium/third_party/webrtc/modules/desktop_capture/mock_desktop_capturer_callback.h +++ b/chromium/third_party/webrtc/modules/desktop_capture/mock_desktop_capturer_callback.h @@ -22,9 +22,10 @@ class MockDesktopCapturerCallback : public DesktopCapturer::Callback { MockDesktopCapturerCallback(); ~MockDesktopCapturerCallback() override; - MOCK_METHOD2(OnCaptureResultPtr, - void(DesktopCapturer::Result result, - std::unique_ptr<DesktopFrame>* frame)); + MOCK_METHOD(void, + OnCaptureResultPtr, + (DesktopCapturer::Result result, + std::unique_ptr<DesktopFrame>* frame)); void OnCaptureResult(DesktopCapturer::Result result, std::unique_ptr<DesktopFrame> frame) final; diff --git a/chromium/third_party/webrtc/modules/pacing/BUILD.gn b/chromium/third_party/webrtc/modules/pacing/BUILD.gn index 6f65c33942e..b19c304e1f6 100644 --- a/chromium/third_party/webrtc/modules/pacing/BUILD.gn +++ b/chromium/third_party/webrtc/modules/pacing/BUILD.gn @@ -49,6 +49,7 @@ rtc_library("pacing") { "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_task_queue", "../../rtc_base/experiments:field_trial_parser", + "../../rtc_base/synchronization:mutex", "../../rtc_base/synchronization:sequence_checker", "../../rtc_base/task_utils:to_queued_task", "../../system_wrappers", @@ -57,6 +58,8 @@ rtc_library("pacing") { "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", "../utility", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", diff --git a/chromium/third_party/webrtc/modules/pacing/paced_sender.cc b/chromium/third_party/webrtc/modules/pacing/paced_sender.cc index cd298f9b0bf..e38863031b3 100644 --- a/chromium/third_party/webrtc/modules/pacing/paced_sender.cc +++ b/chromium/third_party/webrtc/modules/pacing/paced_sender.cc @@ -22,13 +22,15 @@ #include "rtc_base/location.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" #include "system_wrappers/include/clock.h" namespace webrtc { const int64_t PacedSender::kMaxQueueLengthMs = 2000; const float PacedSender::kDefaultPaceMultiplier = 2.5f; -PacedSender::PacedSender(Clock* clock, PacketRouter* packet_router, +PacedSender::PacedSender(Clock* clock, + PacketRouter* packet_router, RtcEventLog* event_log, const WebRtcKeyValueConfig* field_trials, ProcessThread* process_thread) @@ -40,7 +42,9 @@ PacedSender::PacedSender(Clock* clock, PacketRouter* packet_router, : PacingController::ProcessMode::kPeriodic), pacing_controller_(clock, static_cast<PacingController::PacketSender*>(this), - event_log, field_trials, process_mode_), + event_log, + field_trials, + process_mode_), clock_(clock), packet_router_(packet_router), process_thread_(process_thread) { @@ -112,8 +116,15 @@ void PacedSender::SetPacingRates(DataRate pacing_rate, DataRate padding_rate) { void PacedSender::EnqueuePackets( std::vector<std::unique_ptr<RtpPacketToSend>> packets) { { + TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "PacedSender::EnqueuePackets"); rtc::CritScope cs(&critsect_); for (auto& packet : packets) { + TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "PacedSender::EnqueuePackets::Loop", "sequence_number", + packet->SequenceNumber(), "rtp_timestamp", + packet->Timestamp()); + pacing_controller_.EnqueuePacket(std::move(packet)); } } diff --git a/chromium/third_party/webrtc/modules/pacing/paced_sender_unittest.cc b/chromium/third_party/webrtc/modules/pacing/paced_sender_unittest.cc index 26d2eac4132..dcbe7d56556 100644 --- a/chromium/third_party/webrtc/modules/pacing/paced_sender_unittest.cc +++ b/chromium/third_party/webrtc/modules/pacing/paced_sender_unittest.cc @@ -39,12 +39,15 @@ constexpr size_t kDefaultPacketSize = 234; // Mock callback implementing the raw api. class MockCallback : public PacketRouter { public: - MOCK_METHOD2(SendPacket, - void(std::unique_ptr<RtpPacketToSend> packet, - const PacedPacketInfo& cluster_info)); - MOCK_METHOD1( - GeneratePadding, - std::vector<std::unique_ptr<RtpPacketToSend>>(size_t target_size_bytes)); + MOCK_METHOD(void, + SendPacket, + (std::unique_ptr<RtpPacketToSend> packet, + const PacedPacketInfo& cluster_info), + (override)); + MOCK_METHOD(std::vector<std::unique_ptr<RtpPacketToSend>>, + GeneratePadding, + (size_t target_size_bytes), + (override)); }; class ProcessModeTrials : public WebRtcKeyValueConfig { diff --git a/chromium/third_party/webrtc/modules/pacing/pacing_controller.cc b/chromium/third_party/webrtc/modules/pacing/pacing_controller.cc index 4b4fb0bd260..7c523068438 100644 --- a/chromium/third_party/webrtc/modules/pacing/pacing_controller.cc +++ b/chromium/third_party/webrtc/modules/pacing/pacing_controller.cc @@ -193,6 +193,10 @@ bool PacingController::Congested() const { return false; } +bool PacingController::IsProbing() const { + return prober_.is_probing(); +} + Timestamp PacingController::CurrentTime() const { Timestamp time = clock_->CurrentTime(); if (time < last_timestamp_) { diff --git a/chromium/third_party/webrtc/modules/pacing/pacing_controller.h b/chromium/third_party/webrtc/modules/pacing/pacing_controller.h index 27f1614b08c..20d2539e452 100644 --- a/chromium/third_party/webrtc/modules/pacing/pacing_controller.h +++ b/chromium/third_party/webrtc/modules/pacing/pacing_controller.h @@ -146,6 +146,8 @@ class PacingController { bool Congested() const; + bool IsProbing() const; + private: void EnqueuePacketInternal(std::unique_ptr<RtpPacketToSend> packet, int priority); diff --git a/chromium/third_party/webrtc/modules/pacing/pacing_controller_unittest.cc b/chromium/third_party/webrtc/modules/pacing/pacing_controller_unittest.cc index fa23da70a02..e7a61f75e47 100644 --- a/chromium/third_party/webrtc/modules/pacing/pacing_controller_unittest.cc +++ b/chromium/third_party/webrtc/modules/pacing/pacing_controller_unittest.cc @@ -90,24 +90,28 @@ class MockPacingControllerCallback : public PacingController::PacketSender { return ret; } - MOCK_METHOD5(SendPacket, - void(uint32_t ssrc, - uint16_t sequence_number, - int64_t capture_timestamp, - bool retransmission, - bool padding)); - MOCK_METHOD1(SendPadding, size_t(size_t target_size)); + MOCK_METHOD(void, + SendPacket, + (uint32_t ssrc, + uint16_t sequence_number, + int64_t capture_timestamp, + bool retransmission, + bool padding)); + MOCK_METHOD(size_t, SendPadding, (size_t target_size)); }; // Mock callback implementing the raw api. class MockPacketSender : public PacingController::PacketSender { public: - MOCK_METHOD2(SendRtpPacket, - void(std::unique_ptr<RtpPacketToSend> packet, - const PacedPacketInfo& cluster_info)); - MOCK_METHOD1( - GeneratePadding, - std::vector<std::unique_ptr<RtpPacketToSend>>(DataSize target_size)); + MOCK_METHOD(void, + SendRtpPacket, + (std::unique_ptr<RtpPacketToSend> packet, + const PacedPacketInfo& cluster_info), + (override)); + MOCK_METHOD(std::vector<std::unique_ptr<RtpPacketToSend>>, + GeneratePadding, + (DataSize target_size), + (override)); }; class PacingControllerPadding : public PacingController::PacketSender { diff --git a/chromium/third_party/webrtc/modules/pacing/packet_router.cc b/chromium/third_party/webrtc/modules/pacing/packet_router.cc index fa643314934..3569738cdf1 100644 --- a/chromium/third_party/webrtc/modules/pacing/packet_router.cc +++ b/chromium/third_party/webrtc/modules/pacing/packet_router.cc @@ -17,13 +17,14 @@ #include <utility> #include "absl/types/optional.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtcp_packet.h" #include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" +#include "rtc_base/trace_event.h" namespace webrtc { namespace { @@ -52,8 +53,9 @@ PacketRouter::~PacketRouter() { RTC_DCHECK(active_remb_module_ == nullptr); } -void PacketRouter::AddSendRtpModule(RtpRtcp* rtp_module, bool remb_candidate) { - rtc::CritScope cs(&modules_crit_); +void PacketRouter::AddSendRtpModule(RtpRtcpInterface* rtp_module, + bool remb_candidate) { + MutexLock lock(&modules_mutex_); AddSendRtpModuleToMap(rtp_module, rtp_module->SSRC()); if (absl::optional<uint32_t> rtx_ssrc = rtp_module->RtxSsrc()) { @@ -72,7 +74,8 @@ void PacketRouter::AddSendRtpModule(RtpRtcp* rtp_module, bool remb_candidate) { } } -void PacketRouter::AddSendRtpModuleToMap(RtpRtcp* rtp_module, uint32_t ssrc) { +void PacketRouter::AddSendRtpModuleToMap(RtpRtcpInterface* rtp_module, + uint32_t ssrc) { RTC_DCHECK(send_modules_map_.find(ssrc) == send_modules_map_.end()); // Always keep the audio modules at the back of the list, so that when we // iterate over the modules in order to find one that can send padding we @@ -93,8 +96,8 @@ void PacketRouter::RemoveSendRtpModuleFromMap(uint32_t ssrc) { send_modules_map_.erase(kv); } -void PacketRouter::RemoveSendRtpModule(RtpRtcp* rtp_module) { - rtc::CritScope cs(&modules_crit_); +void PacketRouter::RemoveSendRtpModule(RtpRtcpInterface* rtp_module) { + MutexLock lock(&modules_mutex_); MaybeRemoveRembModuleCandidate(rtp_module, /* media_sender = */ true); RemoveSendRtpModuleFromMap(rtp_module->SSRC()); @@ -112,7 +115,7 @@ void PacketRouter::RemoveSendRtpModule(RtpRtcp* rtp_module) { void PacketRouter::AddReceiveRtpModule(RtcpFeedbackSenderInterface* rtcp_sender, bool remb_candidate) { - rtc::CritScope cs(&modules_crit_); + MutexLock lock(&modules_mutex_); RTC_DCHECK(std::find(rtcp_feedback_senders_.begin(), rtcp_feedback_senders_.end(), rtcp_sender) == rtcp_feedback_senders_.end()); @@ -126,7 +129,7 @@ void PacketRouter::AddReceiveRtpModule(RtcpFeedbackSenderInterface* rtcp_sender, void PacketRouter::RemoveReceiveRtpModule( RtcpFeedbackSenderInterface* rtcp_sender) { - rtc::CritScope cs(&modules_crit_); + MutexLock lock(&modules_mutex_); MaybeRemoveRembModuleCandidate(rtcp_sender, /* media_sender = */ false); auto it = std::find(rtcp_feedback_senders_.begin(), rtcp_feedback_senders_.end(), rtcp_sender); @@ -136,7 +139,11 @@ void PacketRouter::RemoveReceiveRtpModule( void PacketRouter::SendPacket(std::unique_ptr<RtpPacketToSend> packet, const PacedPacketInfo& cluster_info) { - rtc::CritScope cs(&modules_crit_); + TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), "PacketRouter::SendPacket", + "sequence_number", packet->SequenceNumber(), "rtp_timestamp", + packet->Timestamp()); + + MutexLock lock(&modules_mutex_); // With the new pacer code path, transport sequence numbers are only set here, // on the pacer thread. Therefore we don't need atomics/synchronization. if (packet->HasExtension<TransportSequenceNumber>()) { @@ -153,7 +160,7 @@ void PacketRouter::SendPacket(std::unique_ptr<RtpPacketToSend> packet, return; } - RtpRtcp* rtp_module = kv->second; + RtpRtcpInterface* rtp_module = kv->second; if (!rtp_module->TrySendPacket(packet.get(), cluster_info)) { RTC_LOG(LS_WARNING) << "Failed to send packet, rejected by RTP module."; return; @@ -168,7 +175,10 @@ void PacketRouter::SendPacket(std::unique_ptr<RtpPacketToSend> packet, std::vector<std::unique_ptr<RtpPacketToSend>> PacketRouter::GeneratePadding( size_t target_size_bytes) { - rtc::CritScope cs(&modules_crit_); + TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "PacketRouter::GeneratePadding", "bytes", target_size_bytes); + + MutexLock lock(&modules_mutex_); // First try on the last rtp module to have sent media. This increases the // the chance that any payload based padding will be useful as it will be // somewhat distributed over modules according the packet rate, even if it @@ -179,29 +189,37 @@ std::vector<std::unique_ptr<RtpPacketToSend>> PacketRouter::GeneratePadding( if (last_send_module_ != nullptr && last_send_module_->SupportsRtxPayloadPadding()) { padding_packets = last_send_module_->GeneratePadding(target_size_bytes); - if (!padding_packets.empty()) { - return padding_packets; - } } - // Iterate over all modules send module. Video modules will be at the front - // and so will be prioritized. This is important since audio packets may not - // be taken into account by the bandwidth estimator, e.g. in FF. - for (RtpRtcp* rtp_module : send_modules_list_) { - if (rtp_module->SupportsPadding()) { - padding_packets = rtp_module->GeneratePadding(target_size_bytes); - if (!padding_packets.empty()) { - last_send_module_ = rtp_module; - break; + if (padding_packets.empty()) { + // Iterate over all modules send module. Video modules will be at the front + // and so will be prioritized. This is important since audio packets may not + // be taken into account by the bandwidth estimator, e.g. in FF. + for (RtpRtcpInterface* rtp_module : send_modules_list_) { + if (rtp_module->SupportsPadding()) { + padding_packets = rtp_module->GeneratePadding(target_size_bytes); + if (!padding_packets.empty()) { + last_send_module_ = rtp_module; + break; + } } } } +#if RTC_TRACE_EVENTS_ENABLED + for (auto& packet : padding_packets) { + TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "PacketRouter::GeneratePadding::Loop", "sequence_number", + packet->SequenceNumber(), "rtp_timestamp", + packet->Timestamp()); + } +#endif + return padding_packets; } uint16_t PacketRouter::CurrentTransportSequenceNumber() const { - rtc::CritScope lock(&modules_crit_); + MutexLock lock(&modules_mutex_); return transport_seq_ & 0xFFFF; } @@ -215,7 +233,7 @@ void PacketRouter::OnReceiveBitrateChanged(const std::vector<uint32_t>& ssrcs, int64_t now_ms = rtc::TimeMillis(); { - rtc::CritScope lock(&remb_crit_); + MutexLock lock(&remb_mutex_); // If we already have an estimate, check if the new total estimate is below // kSendThresholdPercent of the previous estimate. @@ -248,7 +266,7 @@ void PacketRouter::OnReceiveBitrateChanged(const std::vector<uint32_t>& ssrcs, void PacketRouter::SetMaxDesiredReceiveBitrate(int64_t bitrate_bps) { RTC_DCHECK_GE(bitrate_bps, 0); { - rtc::CritScope lock(&remb_crit_); + MutexLock lock(&remb_mutex_); max_bitrate_bps_ = bitrate_bps; if (rtc::TimeMillis() - last_remb_time_ms_ < kRembSendIntervalMs && last_send_bitrate_bps_ > 0 && @@ -262,7 +280,7 @@ void PacketRouter::SetMaxDesiredReceiveBitrate(int64_t bitrate_bps) { bool PacketRouter::SendRemb(int64_t bitrate_bps, const std::vector<uint32_t>& ssrcs) { - rtc::CritScope lock(&modules_crit_); + MutexLock lock(&modules_mutex_); if (!active_remb_module_) { return false; @@ -277,10 +295,10 @@ bool PacketRouter::SendRemb(int64_t bitrate_bps, bool PacketRouter::SendCombinedRtcpPacket( std::vector<std::unique_ptr<rtcp::RtcpPacket>> packets) { - rtc::CritScope cs(&modules_crit_); + MutexLock lock(&modules_mutex_); // Prefer send modules. - for (RtpRtcp* rtp_module : send_modules_list_) { + for (RtpRtcpInterface* rtp_module : send_modules_list_) { if (rtp_module->RTCP() == RtcpMode::kOff) { continue; } diff --git a/chromium/third_party/webrtc/modules/pacing/packet_router.h b/chromium/third_party/webrtc/modules/pacing/packet_router.h index 40b3ad14077..379ec20f200 100644 --- a/chromium/third_party/webrtc/modules/pacing/packet_router.h +++ b/chromium/third_party/webrtc/modules/pacing/packet_router.h @@ -27,11 +27,12 @@ #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/critical_section.h" +#include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" namespace webrtc { -class RtpRtcp; +class RtpRtcpInterface; // PacketRouter keeps track of rtp send modules to support the pacer. // In addition, it handles feedback messages, which are sent on a send @@ -45,8 +46,8 @@ class PacketRouter : public RemoteBitrateObserver, explicit PacketRouter(uint16_t start_transport_seq); ~PacketRouter() override; - void AddSendRtpModule(RtpRtcp* rtp_module, bool remb_candidate); - void RemoveSendRtpModule(RtpRtcp* rtp_module); + void AddSendRtpModule(RtpRtcpInterface* rtp_module, bool remb_candidate); + void RemoveSendRtpModule(RtpRtcpInterface* rtp_module); void AddReceiveRtpModule(RtcpFeedbackSenderInterface* rtcp_sender, bool remb_candidate); @@ -82,48 +83,49 @@ class PacketRouter : public RemoteBitrateObserver, private: void AddRembModuleCandidate(RtcpFeedbackSenderInterface* candidate_module, bool media_sender) - RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_mutex_); void MaybeRemoveRembModuleCandidate( RtcpFeedbackSenderInterface* candidate_module, - bool media_sender) RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_crit_); - void UnsetActiveRembModule() RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_crit_); - void DetermineActiveRembModule() RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_crit_); - void AddSendRtpModuleToMap(RtpRtcp* rtp_module, uint32_t ssrc) - RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_crit_); + bool media_sender) RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_mutex_); + void UnsetActiveRembModule() RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_mutex_); + void DetermineActiveRembModule() RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_mutex_); + void AddSendRtpModuleToMap(RtpRtcpInterface* rtp_module, uint32_t ssrc) + RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_mutex_); void RemoveSendRtpModuleFromMap(uint32_t ssrc) - RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_crit_); - - rtc::CriticalSection modules_crit_; - // Ssrc to RtpRtcp module; - std::unordered_map<uint32_t, RtpRtcp*> send_modules_map_ - RTC_GUARDED_BY(modules_crit_); - std::list<RtpRtcp*> send_modules_list_ RTC_GUARDED_BY(modules_crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(modules_mutex_); + + mutable Mutex modules_mutex_; + // Ssrc to RtpRtcpInterface module; + std::unordered_map<uint32_t, RtpRtcpInterface*> send_modules_map_ + RTC_GUARDED_BY(modules_mutex_); + std::list<RtpRtcpInterface*> send_modules_list_ + RTC_GUARDED_BY(modules_mutex_); // The last module used to send media. - RtpRtcp* last_send_module_ RTC_GUARDED_BY(modules_crit_); + RtpRtcpInterface* last_send_module_ RTC_GUARDED_BY(modules_mutex_); // Rtcp modules of the rtp receivers. std::vector<RtcpFeedbackSenderInterface*> rtcp_feedback_senders_ - RTC_GUARDED_BY(modules_crit_); + RTC_GUARDED_BY(modules_mutex_); - // TODO(eladalon): remb_crit_ only ever held from one function, and it's not + // TODO(eladalon): remb_mutex_ only ever held from one function, and it's not // clear if that function can actually be called from more than one thread. - rtc::CriticalSection remb_crit_; + Mutex remb_mutex_; // The last time a REMB was sent. - int64_t last_remb_time_ms_ RTC_GUARDED_BY(remb_crit_); - int64_t last_send_bitrate_bps_ RTC_GUARDED_BY(remb_crit_); + int64_t last_remb_time_ms_ RTC_GUARDED_BY(remb_mutex_); + int64_t last_send_bitrate_bps_ RTC_GUARDED_BY(remb_mutex_); // The last bitrate update. - int64_t bitrate_bps_ RTC_GUARDED_BY(remb_crit_); - int64_t max_bitrate_bps_ RTC_GUARDED_BY(remb_crit_); + int64_t bitrate_bps_ RTC_GUARDED_BY(remb_mutex_); + int64_t max_bitrate_bps_ RTC_GUARDED_BY(remb_mutex_); // Candidates for the REMB module can be RTP sender/receiver modules, with // the sender modules taking precedence. std::vector<RtcpFeedbackSenderInterface*> sender_remb_candidates_ - RTC_GUARDED_BY(modules_crit_); + RTC_GUARDED_BY(modules_mutex_); std::vector<RtcpFeedbackSenderInterface*> receiver_remb_candidates_ - RTC_GUARDED_BY(modules_crit_); + RTC_GUARDED_BY(modules_mutex_); RtcpFeedbackSenderInterface* active_remb_module_ - RTC_GUARDED_BY(modules_crit_); + RTC_GUARDED_BY(modules_mutex_); - uint64_t transport_seq_ RTC_GUARDED_BY(modules_crit_); + uint64_t transport_seq_ RTC_GUARDED_BY(modules_mutex_); RTC_DISALLOW_COPY_AND_ASSIGN(PacketRouter); }; diff --git a/chromium/third_party/webrtc/modules/pacing/packet_router_unittest.cc b/chromium/third_party/webrtc/modules/pacing/packet_router_unittest.cc index b8f16cb9242..6af7529e861 100644 --- a/chromium/third_party/webrtc/modules/pacing/packet_router_unittest.cc +++ b/chromium/third_party/webrtc/modules/pacing/packet_router_unittest.cc @@ -101,12 +101,12 @@ TEST_F(PacketRouterTest, GeneratePaddingPrioritizesRtx) { const uint16_t kSsrc1 = 1234; const uint16_t kSsrc2 = 4567; - NiceMock<MockRtpRtcp> rtp_1; + NiceMock<MockRtpRtcpInterface> rtp_1; ON_CALL(rtp_1, RtxSendStatus()).WillByDefault(Return(kRtxRedundantPayloads)); ON_CALL(rtp_1, SSRC()).WillByDefault(Return(kSsrc1)); ON_CALL(rtp_1, SupportsPadding).WillByDefault(Return(false)); - NiceMock<MockRtpRtcp> rtp_2; + NiceMock<MockRtpRtcpInterface> rtp_2; ON_CALL(rtp_2, RtxSendStatus()).WillByDefault(Return(kRtxOff)); ON_CALL(rtp_2, SSRC()).WillByDefault(Return(kSsrc2)); ON_CALL(rtp_2, SupportsPadding).WillByDefault(Return(true)); @@ -142,13 +142,13 @@ TEST_F(PacketRouterTest, GeneratePaddingPrioritizesVideo) { kExpectedPaddingPackets); }; - NiceMock<MockRtpRtcp> audio_module; + NiceMock<MockRtpRtcpInterface> audio_module; ON_CALL(audio_module, RtxSendStatus()).WillByDefault(Return(kRtxOff)); ON_CALL(audio_module, SSRC()).WillByDefault(Return(kSsrc1)); ON_CALL(audio_module, SupportsPadding).WillByDefault(Return(true)); ON_CALL(audio_module, IsAudioConfigured).WillByDefault(Return(true)); - NiceMock<MockRtpRtcp> video_module; + NiceMock<MockRtpRtcpInterface> video_module; ON_CALL(video_module, RtxSendStatus()).WillByDefault(Return(kRtxOff)); ON_CALL(video_module, SSRC()).WillByDefault(Return(kSsrc2)); ON_CALL(video_module, SupportsPadding).WillByDefault(Return(true)); @@ -194,7 +194,7 @@ TEST_F(PacketRouterTest, PadsOnLastActiveMediaStream) { const uint16_t kSsrc3 = 8901; // First two rtp modules send media and have rtx. - NiceMock<MockRtpRtcp> rtp_1; + NiceMock<MockRtpRtcpInterface> rtp_1; EXPECT_CALL(rtp_1, SSRC()).WillRepeatedly(Return(kSsrc1)); EXPECT_CALL(rtp_1, SupportsPadding).WillRepeatedly(Return(true)); EXPECT_CALL(rtp_1, SupportsRtxPayloadPadding).WillRepeatedly(Return(true)); @@ -205,7 +205,7 @@ TEST_F(PacketRouterTest, PadsOnLastActiveMediaStream) { ::testing::Pointee(Property(&RtpPacketToSend::Ssrc, kSsrc1)), _)) .WillRepeatedly(Return(true)); - NiceMock<MockRtpRtcp> rtp_2; + NiceMock<MockRtpRtcpInterface> rtp_2; EXPECT_CALL(rtp_2, SSRC()).WillRepeatedly(Return(kSsrc2)); EXPECT_CALL(rtp_2, SupportsPadding).WillRepeatedly(Return(true)); EXPECT_CALL(rtp_2, SupportsRtxPayloadPadding).WillRepeatedly(Return(true)); @@ -217,7 +217,7 @@ TEST_F(PacketRouterTest, PadsOnLastActiveMediaStream) { .WillRepeatedly(Return(true)); // Third module is sending media, but does not support rtx. - NiceMock<MockRtpRtcp> rtp_3; + NiceMock<MockRtpRtcpInterface> rtp_3; EXPECT_CALL(rtp_3, SSRC()).WillRepeatedly(Return(kSsrc3)); EXPECT_CALL(rtp_3, SupportsPadding).WillRepeatedly(Return(true)); EXPECT_CALL(rtp_3, SupportsRtxPayloadPadding).WillRepeatedly(Return(false)); @@ -265,7 +265,7 @@ TEST_F(PacketRouterTest, PadsOnLastActiveMediaStream) { packet_router_.RemoveSendRtpModule(&rtp_2); // Send on and then remove all remaining modules. - RtpRtcp* last_send_module; + RtpRtcpInterface* last_send_module; EXPECT_CALL(rtp_1, GeneratePadding(kPaddingBytes)) .Times(1) .WillOnce([&](size_t target_size_bytes) { @@ -297,7 +297,7 @@ TEST_F(PacketRouterTest, AllocatesTransportSequenceNumbers) { const uint16_t kSsrc1 = 1234; PacketRouter packet_router(kStartSeq - 1); - NiceMock<MockRtpRtcp> rtp_1; + NiceMock<MockRtpRtcpInterface> rtp_1; EXPECT_CALL(rtp_1, SSRC()).WillRepeatedly(Return(kSsrc1)); EXPECT_CALL(rtp_1, TrySendPacket).WillRepeatedly(Return(true)); packet_router.AddSendRtpModule(&rtp_1, false); @@ -315,8 +315,8 @@ TEST_F(PacketRouterTest, AllocatesTransportSequenceNumbers) { } TEST_F(PacketRouterTest, SendTransportFeedback) { - NiceMock<MockRtpRtcp> rtp_1; - NiceMock<MockRtpRtcp> rtp_2; + NiceMock<MockRtpRtcpInterface> rtp_1; + NiceMock<MockRtpRtcpInterface> rtp_2; ON_CALL(rtp_1, RTCP()).WillByDefault(Return(RtcpMode::kCompound)); ON_CALL(rtp_2, RTCP()).WillByDefault(Return(RtcpMode::kCompound)); @@ -338,7 +338,7 @@ TEST_F(PacketRouterTest, SendTransportFeedback) { TEST_F(PacketRouterTest, SendPacketWithoutTransportSequenceNumbers) { const uint16_t kSsrc1 = 1234; - NiceMock<MockRtpRtcp> rtp_1; + NiceMock<MockRtpRtcpInterface> rtp_1; ON_CALL(rtp_1, SendingMedia).WillByDefault(Return(true)); ON_CALL(rtp_1, SSRC).WillByDefault(Return(kSsrc1)); packet_router_.AddSendRtpModule(&rtp_1, false); @@ -361,8 +361,8 @@ TEST_F(PacketRouterTest, SendPacketWithoutTransportSequenceNumbers) { } TEST_F(PacketRouterTest, SendPacketAssignsTransportSequenceNumbers) { - NiceMock<MockRtpRtcp> rtp_1; - NiceMock<MockRtpRtcp> rtp_2; + NiceMock<MockRtpRtcpInterface> rtp_1; + NiceMock<MockRtpRtcpInterface> rtp_2; const uint16_t kSsrc1 = 1234; const uint16_t kSsrc2 = 2345; @@ -405,8 +405,9 @@ TEST_F(PacketRouterTest, SendPacketAssignsTransportSequenceNumbers) { } #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID) -TEST_F(PacketRouterTest, DoubleRegistrationOfSendModuleDisallowed) { - NiceMock<MockRtpRtcp> module; +using PacketRouterDeathTest = PacketRouterTest; +TEST_F(PacketRouterDeathTest, DoubleRegistrationOfSendModuleDisallowed) { + NiceMock<MockRtpRtcpInterface> module; constexpr bool remb_candidate = false; // Value irrelevant. packet_router_.AddSendRtpModule(&module, remb_candidate); @@ -416,8 +417,8 @@ TEST_F(PacketRouterTest, DoubleRegistrationOfSendModuleDisallowed) { packet_router_.RemoveSendRtpModule(&module); } -TEST_F(PacketRouterTest, DoubleRegistrationOfReceiveModuleDisallowed) { - NiceMock<MockRtpRtcp> module; +TEST_F(PacketRouterDeathTest, DoubleRegistrationOfReceiveModuleDisallowed) { + NiceMock<MockRtpRtcpInterface> module; constexpr bool remb_candidate = false; // Value irrelevant. packet_router_.AddReceiveRtpModule(&module, remb_candidate); @@ -427,14 +428,14 @@ TEST_F(PacketRouterTest, DoubleRegistrationOfReceiveModuleDisallowed) { packet_router_.RemoveReceiveRtpModule(&module); } -TEST_F(PacketRouterTest, RemovalOfNeverAddedSendModuleDisallowed) { - NiceMock<MockRtpRtcp> module; +TEST_F(PacketRouterDeathTest, RemovalOfNeverAddedSendModuleDisallowed) { + NiceMock<MockRtpRtcpInterface> module; EXPECT_DEATH(packet_router_.RemoveSendRtpModule(&module), ""); } -TEST_F(PacketRouterTest, RemovalOfNeverAddedReceiveModuleDisallowed) { - NiceMock<MockRtpRtcp> module; +TEST_F(PacketRouterDeathTest, RemovalOfNeverAddedReceiveModuleDisallowed) { + NiceMock<MockRtpRtcpInterface> module; EXPECT_DEATH(packet_router_.RemoveReceiveRtpModule(&module), ""); } @@ -442,7 +443,7 @@ TEST_F(PacketRouterTest, RemovalOfNeverAddedReceiveModuleDisallowed) { TEST(PacketRouterRembTest, LowerEstimateToSendRemb) { rtc::ScopedFakeClock clock; - NiceMock<MockRtpRtcp> rtp; + NiceMock<MockRtpRtcpInterface> rtp; PacketRouter packet_router; packet_router.AddSendRtpModule(&rtp, true); @@ -468,7 +469,7 @@ TEST(PacketRouterRembTest, LowerEstimateToSendRemb) { TEST(PacketRouterRembTest, VerifyIncreasingAndDecreasing) { rtc::ScopedFakeClock clock; - NiceMock<MockRtpRtcp> rtp; + NiceMock<MockRtpRtcpInterface> rtp; PacketRouter packet_router; packet_router.AddSendRtpModule(&rtp, true); @@ -493,7 +494,7 @@ TEST(PacketRouterRembTest, VerifyIncreasingAndDecreasing) { TEST(PacketRouterRembTest, NoRembForIncreasedBitrate) { rtc::ScopedFakeClock clock; - NiceMock<MockRtpRtcp> rtp; + NiceMock<MockRtpRtcpInterface> rtp; PacketRouter packet_router; packet_router.AddSendRtpModule(&rtp, true); @@ -521,8 +522,8 @@ TEST(PacketRouterRembTest, NoRembForIncreasedBitrate) { TEST(PacketRouterRembTest, ChangeSendRtpModule) { rtc::ScopedFakeClock clock; - NiceMock<MockRtpRtcp> rtp_send; - NiceMock<MockRtpRtcp> rtp_recv; + NiceMock<MockRtpRtcpInterface> rtp_send; + NiceMock<MockRtpRtcpInterface> rtp_recv; PacketRouter packet_router; packet_router.AddSendRtpModule(&rtp_send, true); packet_router.AddReceiveRtpModule(&rtp_recv, true); @@ -556,7 +557,7 @@ TEST(PacketRouterRembTest, ChangeSendRtpModule) { TEST(PacketRouterRembTest, OnlyOneRembForRepeatedOnReceiveBitrateChanged) { rtc::ScopedFakeClock clock; - NiceMock<MockRtpRtcp> rtp; + NiceMock<MockRtpRtcpInterface> rtp; PacketRouter packet_router; packet_router.AddSendRtpModule(&rtp, true); @@ -585,7 +586,7 @@ TEST(PacketRouterRembTest, SetMaxDesiredReceiveBitrateLimitsSetRemb) { rtc::ScopedFakeClock clock; PacketRouter packet_router; clock.AdvanceTime(TimeDelta::Millis(1000)); - NiceMock<MockRtpRtcp> remb_sender; + NiceMock<MockRtpRtcpInterface> remb_sender; constexpr bool remb_candidate = true; packet_router.AddSendRtpModule(&remb_sender, remb_candidate); @@ -608,7 +609,7 @@ TEST(PacketRouterRembTest, rtc::ScopedFakeClock clock; PacketRouter packet_router; clock.AdvanceTime(TimeDelta::Millis(1000)); - NiceMock<MockRtpRtcp> remb_sender; + NiceMock<MockRtpRtcpInterface> remb_sender; constexpr bool remb_candidate = true; packet_router.AddSendRtpModule(&remb_sender, remb_candidate); @@ -630,7 +631,7 @@ TEST(PacketRouterRembTest, rtc::ScopedFakeClock clock; PacketRouter packet_router; clock.AdvanceTime(TimeDelta::Millis(1000)); - NiceMock<MockRtpRtcp> remb_sender; + NiceMock<MockRtpRtcpInterface> remb_sender; constexpr bool remb_candidate = true; packet_router.AddSendRtpModule(&remb_sender, remb_candidate); @@ -652,7 +653,7 @@ TEST(PacketRouterRembTest, rtc::ScopedFakeClock clock; PacketRouter packet_router; clock.AdvanceTime(TimeDelta::Millis(1000)); - NiceMock<MockRtpRtcp> remb_sender; + NiceMock<MockRtpRtcpInterface> remb_sender; constexpr bool remb_candidate = true; packet_router.AddSendRtpModule(&remb_sender, remb_candidate); @@ -674,7 +675,7 @@ TEST(PacketRouterRembTest, rtc::ScopedFakeClock clock; PacketRouter packet_router; clock.AdvanceTime(TimeDelta::Millis(1000)); - NiceMock<MockRtpRtcp> remb_sender; + NiceMock<MockRtpRtcpInterface> remb_sender; constexpr bool remb_candidate = true; packet_router.AddSendRtpModule(&remb_sender, remb_candidate); @@ -697,7 +698,7 @@ TEST(PacketRouterRembTest, rtc::ScopedFakeClock clock; PacketRouter packet_router; clock.AdvanceTime(TimeDelta::Millis(1000)); - NiceMock<MockRtpRtcp> remb_sender; + NiceMock<MockRtpRtcpInterface> remb_sender; constexpr bool remb_candidate = true; packet_router.AddSendRtpModule(&remb_sender, remb_candidate); @@ -719,7 +720,7 @@ TEST(PacketRouterRembTest, // packet on this one. TEST(PacketRouterRembTest, NoSendingRtpModule) { rtc::ScopedFakeClock clock; - NiceMock<MockRtpRtcp> rtp; + NiceMock<MockRtpRtcpInterface> rtp; PacketRouter packet_router; packet_router.AddReceiveRtpModule(&rtp, true); @@ -745,7 +746,7 @@ TEST(PacketRouterRembTest, NoSendingRtpModule) { TEST(PacketRouterRembTest, NonCandidateSendRtpModuleNotUsedForRemb) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> module; + NiceMock<MockRtpRtcpInterface> module; constexpr bool remb_candidate = false; @@ -764,7 +765,7 @@ TEST(PacketRouterRembTest, NonCandidateSendRtpModuleNotUsedForRemb) { TEST(PacketRouterRembTest, CandidateSendRtpModuleUsedForRemb) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> module; + NiceMock<MockRtpRtcpInterface> module; constexpr bool remb_candidate = true; @@ -783,7 +784,7 @@ TEST(PacketRouterRembTest, CandidateSendRtpModuleUsedForRemb) { TEST(PacketRouterRembTest, NonCandidateReceiveRtpModuleNotUsedForRemb) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> module; + NiceMock<MockRtpRtcpInterface> module; constexpr bool remb_candidate = false; @@ -802,7 +803,7 @@ TEST(PacketRouterRembTest, NonCandidateReceiveRtpModuleNotUsedForRemb) { TEST(PacketRouterRembTest, CandidateReceiveRtpModuleUsedForRemb) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> module; + NiceMock<MockRtpRtcpInterface> module; constexpr bool remb_candidate = true; @@ -822,8 +823,8 @@ TEST(PacketRouterRembTest, SendCandidatePreferredOverReceiveCandidate_SendModuleAddedFirst) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> send_module; - NiceMock<MockRtpRtcp> receive_module; + NiceMock<MockRtpRtcpInterface> send_module; + NiceMock<MockRtpRtcpInterface> receive_module; constexpr bool remb_candidate = true; @@ -850,8 +851,8 @@ TEST(PacketRouterRembTest, SendCandidatePreferredOverReceiveCandidate_ReceiveModuleAddedFirst) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> send_module; - NiceMock<MockRtpRtcp> receive_module; + NiceMock<MockRtpRtcpInterface> send_module; + NiceMock<MockRtpRtcpInterface> receive_module; constexpr bool remb_candidate = true; @@ -877,8 +878,8 @@ TEST(PacketRouterRembTest, TEST(PacketRouterRembTest, ReceiveModuleTakesOverWhenLastSendModuleRemoved) { rtc::ScopedFakeClock clock; PacketRouter packet_router; - NiceMock<MockRtpRtcp> send_module; - NiceMock<MockRtpRtcp> receive_module; + NiceMock<MockRtpRtcpInterface> send_module; + NiceMock<MockRtpRtcpInterface> receive_module; constexpr bool remb_candidate = true; diff --git a/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.cc b/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.cc index 16d6df554c2..531e9d6ad32 100644 --- a/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.cc +++ b/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.cc @@ -17,6 +17,7 @@ #include "rtc_base/event.h" #include "rtc_base/logging.h" #include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/trace_event.h" namespace webrtc { namespace { @@ -34,8 +35,10 @@ TaskQueuePacedSender::TaskQueuePacedSender( PacketRouter* packet_router, RtcEventLog* event_log, const WebRtcKeyValueConfig* field_trials, - TaskQueueFactory* task_queue_factory) + TaskQueueFactory* task_queue_factory, + TimeDelta hold_back_window) : clock_(clock), + hold_back_window_(hold_back_window), packet_router_(packet_router), pacing_controller_(clock, static_cast<PacingController::PacketSender*>(this), @@ -120,6 +123,17 @@ void TaskQueuePacedSender::SetPacingRates(DataRate pacing_rate, void TaskQueuePacedSender::EnqueuePackets( std::vector<std::unique_ptr<RtpPacketToSend>> packets) { +#if RTC_TRACE_EVENTS_ENABLED + TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "TaskQueuePacedSender::EnqueuePackets"); + for (auto& packet : packets) { + TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("webrtc"), + "TaskQueuePacedSender::EnqueuePackets::Loop", + "sequence_number", packet->SequenceNumber(), "rtp_timestamp", + packet->Timestamp()); + } +#endif + task_queue_.PostTask([this, packets_ = std::move(packets)]() mutable { RTC_DCHECK_RUN_ON(&task_queue_); for (auto& packet : packets_) { @@ -175,7 +189,7 @@ TimeDelta TaskQueuePacedSender::OldestPacketWaitTime() const { } void TaskQueuePacedSender::OnStatsUpdated(const Stats& stats) { - rtc::CritScope cs(&stats_crit_); + MutexLock lock(&stats_mutex_); current_stats_ = stats; } @@ -205,8 +219,10 @@ void TaskQueuePacedSender::MaybeProcessPackets( next_process_time = pacing_controller_.NextSendTime(); } - next_process_time = - std::max(now + PacingController::kMinSleepTime, next_process_time); + const TimeDelta min_sleep = pacing_controller_.IsProbing() + ? PacingController::kMinSleepTime + : hold_back_window_; + next_process_time = std::max(now + min_sleep, next_process_time); TimeDelta sleep_time = next_process_time - now; if (next_process_time_.IsMinusInfinity() || @@ -295,7 +311,7 @@ void TaskQueuePacedSender::MaybeUpdateStats(bool is_scheduled_call) { } TaskQueuePacedSender::Stats TaskQueuePacedSender::GetStats() const { - rtc::CritScope cs(&stats_crit_); + MutexLock lock(&stats_mutex_); return current_stats_; } diff --git a/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.h b/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.h index 3f53f000970..71b3be27e6f 100644 --- a/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.h +++ b/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender.h @@ -30,6 +30,7 @@ #include "modules/pacing/rtp_packet_pacer.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" #include "rtc_base/critical_section.h" +#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/task_queue.h" #include "rtc_base/thread_annotations.h" @@ -42,11 +43,18 @@ class TaskQueuePacedSender : public RtpPacketPacer, public RtpPacketSender, private PacingController::PacketSender { public: - TaskQueuePacedSender(Clock* clock, - PacketRouter* packet_router, - RtcEventLog* event_log, - const WebRtcKeyValueConfig* field_trials, - TaskQueueFactory* task_queue_factory); + // The |hold_back_window| parameter sets a lower bound on time to sleep if + // there is currently a pacer queue and packets can't immediately be + // processed. Increasing this reduces thread wakeups at the expense of higher + // latency. + // TODO(bugs.webrtc.org/10809): Remove default value for hold_back_window. + TaskQueuePacedSender( + Clock* clock, + PacketRouter* packet_router, + RtcEventLog* event_log, + const WebRtcKeyValueConfig* field_trials, + TaskQueueFactory* task_queue_factory, + TimeDelta hold_back_window = PacingController::kMinSleepTime); ~TaskQueuePacedSender() override; @@ -134,6 +142,7 @@ class TaskQueuePacedSender : public RtpPacketPacer, Stats GetStats() const; Clock* const clock_; + const TimeDelta hold_back_window_; PacketRouter* const packet_router_ RTC_GUARDED_BY(task_queue_); PacingController pacing_controller_ RTC_GUARDED_BY(task_queue_); @@ -159,8 +168,8 @@ class TaskQueuePacedSender : public RtpPacketPacer, // never drain. bool is_shutdown_ RTC_GUARDED_BY(task_queue_); - rtc::CriticalSection stats_crit_; - Stats current_stats_ RTC_GUARDED_BY(stats_crit_); + mutable Mutex stats_mutex_; + Stats current_stats_ RTC_GUARDED_BY(stats_mutex_); rtc::TaskQueue task_queue_; }; diff --git a/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender_unittest.cc b/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender_unittest.cc index 83aa73e9aaa..ab6a24ba42b 100644 --- a/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender_unittest.cc +++ b/chromium/third_party/webrtc/modules/pacing/task_queue_paced_sender_unittest.cc @@ -24,6 +24,7 @@ #include "test/time_controller/simulated_time_controller.h" using ::testing::_; +using ::testing::AtLeast; using ::testing::Return; using ::testing::SaveArg; @@ -37,26 +38,40 @@ constexpr size_t kDefaultPacketSize = 1234; class MockPacketRouter : public PacketRouter { public: - MOCK_METHOD2(SendPacket, - void(std::unique_ptr<RtpPacketToSend> packet, - const PacedPacketInfo& cluster_info)); - MOCK_METHOD1( - GeneratePadding, - std::vector<std::unique_ptr<RtpPacketToSend>>(size_t target_size_bytes)); + MOCK_METHOD(void, + SendPacket, + (std::unique_ptr<RtpPacketToSend> packet, + const PacedPacketInfo& cluster_info), + (override)); + MOCK_METHOD(std::vector<std::unique_ptr<RtpPacketToSend>>, + GeneratePadding, + (size_t target_size_bytes), + (override)); +}; + +class StatsUpdateObserver { + public: + StatsUpdateObserver() = default; + virtual ~StatsUpdateObserver() = default; + + virtual void OnStatsUpdated() = 0; }; class TaskQueuePacedSenderForTest : public TaskQueuePacedSender { public: - TaskQueuePacedSenderForTest(Clock* clock, - PacketRouter* packet_router, - RtcEventLog* event_log, - const WebRtcKeyValueConfig* field_trials, - TaskQueueFactory* task_queue_factory) + TaskQueuePacedSenderForTest( + Clock* clock, + PacketRouter* packet_router, + RtcEventLog* event_log, + const WebRtcKeyValueConfig* field_trials, + TaskQueueFactory* task_queue_factory, + TimeDelta hold_back_window = PacingController::kMinSleepTime) : TaskQueuePacedSender(clock, packet_router, event_log, field_trials, - task_queue_factory) {} + task_queue_factory, + hold_back_window) {} void OnStatsUpdated(const Stats& stats) override { ++num_stats_updates_; @@ -65,250 +80,327 @@ class TaskQueuePacedSenderForTest : public TaskQueuePacedSender { size_t num_stats_updates_ = 0; }; +} // namespace -std::unique_ptr<RtpPacketToSend> BuildRtpPacket(RtpPacketMediaType type) { - auto packet = std::make_unique<RtpPacketToSend>(nullptr); - packet->set_packet_type(type); - switch (type) { - case RtpPacketMediaType::kAudio: - packet->SetSsrc(kAudioSsrc); - break; - case RtpPacketMediaType::kVideo: - packet->SetSsrc(kVideoSsrc); - break; - case RtpPacketMediaType::kRetransmission: - case RtpPacketMediaType::kPadding: - packet->SetSsrc(kVideoRtxSsrc); - break; - case RtpPacketMediaType::kForwardErrorCorrection: - packet->SetSsrc(kFlexFecSsrc); - break; +namespace test { + + std::unique_ptr<RtpPacketToSend> BuildRtpPacket(RtpPacketMediaType type) { + auto packet = std::make_unique<RtpPacketToSend>(nullptr); + packet->set_packet_type(type); + switch (type) { + case RtpPacketMediaType::kAudio: + packet->SetSsrc(kAudioSsrc); + break; + case RtpPacketMediaType::kVideo: + packet->SetSsrc(kVideoSsrc); + break; + case RtpPacketMediaType::kRetransmission: + case RtpPacketMediaType::kPadding: + packet->SetSsrc(kVideoRtxSsrc); + break; + case RtpPacketMediaType::kForwardErrorCorrection: + packet->SetSsrc(kFlexFecSsrc); + break; + } + + packet->SetPayloadSize(kDefaultPacketSize); + return packet; } - packet->SetPayloadSize(kDefaultPacketSize); - return packet; -} + std::vector<std::unique_ptr<RtpPacketToSend>> GeneratePackets( + RtpPacketMediaType type, + size_t num_packets) { + std::vector<std::unique_ptr<RtpPacketToSend>> packets; + for (size_t i = 0; i < num_packets; ++i) { + packets.push_back(BuildRtpPacket(type)); + } + return packets; + } -std::vector<std::unique_ptr<RtpPacketToSend>> GeneratePackets( - RtpPacketMediaType type, - size_t num_packets) { - std::vector<std::unique_ptr<RtpPacketToSend>> packets; - for (size_t i = 0; i < num_packets; ++i) { - packets.push_back(BuildRtpPacket(type)); + TEST(TaskQueuePacedSenderTest, PacesPackets) { + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + PacingController::kMinSleepTime); + + // Insert a number of packets, covering one second. + static constexpr size_t kPacketsToSend = 42; + pacer.SetPacingRates( + DataRate::BitsPerSec(kDefaultPacketSize * 8 * kPacketsToSend), + DataRate::Zero()); + pacer.EnqueuePackets( + GeneratePackets(RtpPacketMediaType::kVideo, kPacketsToSend)); + + // Expect all of them to be sent. + size_t packets_sent = 0; + Timestamp end_time = Timestamp::PlusInfinity(); + EXPECT_CALL(packet_router, SendPacket) + .WillRepeatedly([&](std::unique_ptr<RtpPacketToSend> packet, + const PacedPacketInfo& cluster_info) { + ++packets_sent; + if (packets_sent == kPacketsToSend) { + end_time = time_controller.GetClock()->CurrentTime(); + } + }); + + const Timestamp start_time = time_controller.GetClock()->CurrentTime(); + + // Packets should be sent over a period of close to 1s. Expect a little + // lower than this since initial probing is a bit quicker. + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + EXPECT_EQ(packets_sent, kPacketsToSend); + ASSERT_TRUE(end_time.IsFinite()); + EXPECT_NEAR((end_time - start_time).ms<double>(), 1000.0, 50.0); } - return packets; -} -} // namespace -namespace test { + TEST(TaskQueuePacedSenderTest, ReschedulesProcessOnRateChange) { + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + PacingController::kMinSleepTime); + + // Insert a number of packets to be sent 200ms apart. + const size_t kPacketsPerSecond = 5; + const DataRate kPacingRate = + DataRate::BitsPerSec(kDefaultPacketSize * 8 * kPacketsPerSecond); + pacer.SetPacingRates(kPacingRate, DataRate::Zero()); + + // Send some initial packets to be rid of any probes. + EXPECT_CALL(packet_router, SendPacket).Times(kPacketsPerSecond); + pacer.EnqueuePackets( + GeneratePackets(RtpPacketMediaType::kVideo, kPacketsPerSecond)); + time_controller.AdvanceTime(TimeDelta::Seconds(1)); + + // Insert three packets, and record send time of each of them. + // After the second packet is sent, double the send rate so we can + // check the third packets is sent after half the wait time. + Timestamp first_packet_time = Timestamp::MinusInfinity(); + Timestamp second_packet_time = Timestamp::MinusInfinity(); + Timestamp third_packet_time = Timestamp::MinusInfinity(); + + EXPECT_CALL(packet_router, SendPacket) + .Times(3) + .WillRepeatedly([&](std::unique_ptr<RtpPacketToSend> packet, + const PacedPacketInfo& cluster_info) { + if (first_packet_time.IsInfinite()) { + first_packet_time = time_controller.GetClock()->CurrentTime(); + } else if (second_packet_time.IsInfinite()) { + second_packet_time = time_controller.GetClock()->CurrentTime(); + pacer.SetPacingRates(2 * kPacingRate, DataRate::Zero()); + } else { + third_packet_time = time_controller.GetClock()->CurrentTime(); + } + }); + + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 3)); + time_controller.AdvanceTime(TimeDelta::Millis(500)); + ASSERT_TRUE(third_packet_time.IsFinite()); + EXPECT_NEAR((second_packet_time - first_packet_time).ms<double>(), 200.0, + 1.0); + EXPECT_NEAR((third_packet_time - second_packet_time).ms<double>(), 100.0, + 1.0); + } -class TaskQueuePacedSenderTest : public ::testing::Test { - public: - TaskQueuePacedSenderTest() - : time_controller_(Timestamp::Millis(1234)), - pacer_(time_controller_.GetClock(), - &packet_router_, - /*event_log=*/nullptr, - /*field_trials=*/nullptr, - time_controller_.GetTaskQueueFactory()) {} - - protected: - Timestamp CurrentTime() { return time_controller_.GetClock()->CurrentTime(); } - - GlobalSimulatedTimeController time_controller_; - MockPacketRouter packet_router_; - TaskQueuePacedSender pacer_; -}; + TEST(TaskQueuePacedSenderTest, SendsAudioImmediately) { + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + PacingController::kMinSleepTime); -TEST_F(TaskQueuePacedSenderTest, PacesPackets) { - // Insert a number of packets, covering one second. - static constexpr size_t kPacketsToSend = 42; - pacer_.SetPacingRates( - DataRate::BitsPerSec(kDefaultPacketSize * 8 * kPacketsToSend), - DataRate::Zero()); - pacer_.EnqueuePackets( - GeneratePackets(RtpPacketMediaType::kVideo, kPacketsToSend)); - - // Expect all of them to be sent. - size_t packets_sent = 0; - Timestamp end_time = Timestamp::PlusInfinity(); - EXPECT_CALL(packet_router_, SendPacket) - .WillRepeatedly([&](std::unique_ptr<RtpPacketToSend> packet, - const PacedPacketInfo& cluster_info) { - ++packets_sent; - if (packets_sent == kPacketsToSend) { - end_time = time_controller_.GetClock()->CurrentTime(); - } - }); - - const Timestamp start_time = time_controller_.GetClock()->CurrentTime(); - - // Packets should be sent over a period of close to 1s. Expect a little lower - // than this since initial probing is a bit quicker. - time_controller_.AdvanceTime(TimeDelta::Seconds(1)); - EXPECT_EQ(packets_sent, kPacketsToSend); - ASSERT_TRUE(end_time.IsFinite()); - EXPECT_NEAR((end_time - start_time).ms<double>(), 1000.0, 50.0); -} - -TEST_F(TaskQueuePacedSenderTest, ReschedulesProcessOnRateChange) { - // Insert a number of packets to be sent 200ms apart. - const size_t kPacketsPerSecond = 5; - const DataRate kPacingRate = - DataRate::BitsPerSec(kDefaultPacketSize * 8 * kPacketsPerSecond); - pacer_.SetPacingRates(kPacingRate, DataRate::Zero()); - - // Send some initial packets to be rid of any probes. - EXPECT_CALL(packet_router_, SendPacket).Times(kPacketsPerSecond); - pacer_.EnqueuePackets( - GeneratePackets(RtpPacketMediaType::kVideo, kPacketsPerSecond)); - time_controller_.AdvanceTime(TimeDelta::Seconds(1)); - - // Insert three packets, and record send time of each of them. - // After the second packet is sent, double the send rate so we can - // check the third packets is sent after half the wait time. - Timestamp first_packet_time = Timestamp::MinusInfinity(); - Timestamp second_packet_time = Timestamp::MinusInfinity(); - Timestamp third_packet_time = Timestamp::MinusInfinity(); - - EXPECT_CALL(packet_router_, SendPacket) - .Times(3) - .WillRepeatedly([&](std::unique_ptr<RtpPacketToSend> packet, - const PacedPacketInfo& cluster_info) { - if (first_packet_time.IsInfinite()) { - first_packet_time = CurrentTime(); - } else if (second_packet_time.IsInfinite()) { - second_packet_time = CurrentTime(); - pacer_.SetPacingRates(2 * kPacingRate, DataRate::Zero()); - } else { - third_packet_time = CurrentTime(); - } - }); - - pacer_.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 3)); - time_controller_.AdvanceTime(TimeDelta::Millis(500)); - ASSERT_TRUE(third_packet_time.IsFinite()); - EXPECT_NEAR((second_packet_time - first_packet_time).ms<double>(), 200.0, - 1.0); - EXPECT_NEAR((third_packet_time - second_packet_time).ms<double>(), 100.0, - 1.0); -} - -TEST_F(TaskQueuePacedSenderTest, SendsAudioImmediately) { - const DataRate kPacingDataRate = DataRate::KilobitsPerSec(125); - const DataSize kPacketSize = DataSize::Bytes(kDefaultPacketSize); - const TimeDelta kPacketPacingTime = kPacketSize / kPacingDataRate; - - pacer_.SetPacingRates(kPacingDataRate, DataRate::Zero()); - - // Add some initial video packets, only one should be sent. - EXPECT_CALL(packet_router_, SendPacket); - pacer_.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 10)); - time_controller_.AdvanceTime(TimeDelta::Zero()); - ::testing::Mock::VerifyAndClearExpectations(&packet_router_); - - // Advance time, but still before next packet should be sent. - time_controller_.AdvanceTime(kPacketPacingTime / 2); - - // Insert an audio packet, it should be sent immediately. - EXPECT_CALL(packet_router_, SendPacket); - pacer_.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kAudio, 1)); - time_controller_.AdvanceTime(TimeDelta::Zero()); - ::testing::Mock::VerifyAndClearExpectations(&packet_router_); -} - -TEST(TaskQueuePacedSenderTestNew, RespectedMinTimeBetweenStatsUpdates) { - GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); - MockPacketRouter packet_router; - TaskQueuePacedSenderForTest pacer(time_controller.GetClock(), &packet_router, - /*event_log=*/nullptr, - /*field_trials=*/nullptr, - time_controller.GetTaskQueueFactory()); - const DataRate kPacingDataRate = DataRate::KilobitsPerSec(300); - pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); - - const TimeDelta kMinTimeBetweenStatsUpdates = TimeDelta::Millis(1); - - // Nothing inserted, no stats updates yet. - EXPECT_EQ(pacer.num_stats_updates_, 0u); - - // Insert one packet, stats should be updated. - pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 1)); - time_controller.AdvanceTime(TimeDelta::Zero()); - EXPECT_EQ(pacer.num_stats_updates_, 1u); - - // Advance time half of the min stats update interval, and trigger a - // refresh - stats should not be updated yet. - time_controller.AdvanceTime(kMinTimeBetweenStatsUpdates / 2); - pacer.EnqueuePackets({}); - time_controller.AdvanceTime(TimeDelta::Zero()); - EXPECT_EQ(pacer.num_stats_updates_, 1u); - - // Advance time the next half, now stats update is triggered. - time_controller.AdvanceTime(kMinTimeBetweenStatsUpdates / 2); - pacer.EnqueuePackets({}); - time_controller.AdvanceTime(TimeDelta::Zero()); - EXPECT_EQ(pacer.num_stats_updates_, 2u); -} - -TEST(TaskQueuePacedSenderTestNew, ThrottlesStatsUpdates) { - GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); - MockPacketRouter packet_router; - TaskQueuePacedSenderForTest pacer(time_controller.GetClock(), &packet_router, - /*event_log=*/nullptr, - /*field_trials=*/nullptr, - time_controller.GetTaskQueueFactory()); - - // Set rates so one packet adds 10ms of buffer level. - const DataSize kPacketSize = DataSize::Bytes(kDefaultPacketSize); - const TimeDelta kPacketPacingTime = TimeDelta::Millis(10); - const DataRate kPacingDataRate = kPacketSize / kPacketPacingTime; - const TimeDelta kMinTimeBetweenStatsUpdates = TimeDelta::Millis(1); - const TimeDelta kMaxTimeBetweenStatsUpdates = TimeDelta::Millis(33); - - // Nothing inserted, no stats updates yet. - size_t num_expected_stats_updates = 0; - EXPECT_EQ(pacer.num_stats_updates_, num_expected_stats_updates); - pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); - time_controller.AdvanceTime(kMinTimeBetweenStatsUpdates); - // Updating pacing rates refreshes stats. - EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); - - // Record time when we insert first packet, this triggers the scheduled - // stats updating. - Clock* const clock = time_controller.GetClock(); - const Timestamp start_time = clock->CurrentTime(); - - while (clock->CurrentTime() - start_time <= - kMaxTimeBetweenStatsUpdates - kPacketPacingTime) { - // Enqueue packet, expect stats update. - pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 1)); + const DataRate kPacingDataRate = DataRate::KilobitsPerSec(125); + const DataSize kPacketSize = DataSize::Bytes(kDefaultPacketSize); + const TimeDelta kPacketPacingTime = kPacketSize / kPacingDataRate; + + pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); + + // Add some initial video packets, only one should be sent. + EXPECT_CALL(packet_router, SendPacket); + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 10)); time_controller.AdvanceTime(TimeDelta::Zero()); - EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); + ::testing::Mock::VerifyAndClearExpectations(&packet_router); - // Advance time to halfway through pacing time, expect another stats - // update. + // Advance time, but still before next packet should be sent. time_controller.AdvanceTime(kPacketPacingTime / 2); + + // Insert an audio packet, it should be sent immediately. + EXPECT_CALL(packet_router, SendPacket); + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kAudio, 1)); + time_controller.AdvanceTime(TimeDelta::Zero()); + ::testing::Mock::VerifyAndClearExpectations(&packet_router); + } + + TEST(TaskQueuePacedSenderTest, SleepsDuringCoalscingWindow) { + const TimeDelta kCoalescingWindow = TimeDelta::Millis(5); + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + kCoalescingWindow); + + // Set rates so one packet adds one ms of buffer level. + const DataSize kPacketSize = DataSize::Bytes(kDefaultPacketSize); + const TimeDelta kPacketPacingTime = TimeDelta::Millis(1); + const DataRate kPacingDataRate = kPacketSize / kPacketPacingTime; + + pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); + + // Add 10 packets. The first should be sent immediately since the buffers + // are clear. + EXPECT_CALL(packet_router, SendPacket); + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 10)); + time_controller.AdvanceTime(TimeDelta::Zero()); + ::testing::Mock::VerifyAndClearExpectations(&packet_router); + + // Advance time to 1ms before the coalescing window ends. No packets should + // be sent. + EXPECT_CALL(packet_router, SendPacket).Times(0); + time_controller.AdvanceTime(kCoalescingWindow - TimeDelta::Millis(1)); + + // Advance time to where coalescing window ends. All packets that should + // have been sent up til now will be sent. + EXPECT_CALL(packet_router, SendPacket).Times(5); + time_controller.AdvanceTime(TimeDelta::Millis(1)); + ::testing::Mock::VerifyAndClearExpectations(&packet_router); + } + + TEST(TaskQueuePacedSenderTest, ProbingOverridesCoalescingWindow) { + const TimeDelta kCoalescingWindow = TimeDelta::Millis(5); + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + kCoalescingWindow); + + // Set rates so one packet adds one ms of buffer level. + const DataSize kPacketSize = DataSize::Bytes(kDefaultPacketSize); + const TimeDelta kPacketPacingTime = TimeDelta::Millis(1); + const DataRate kPacingDataRate = kPacketSize / kPacketPacingTime; + + pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); + + // Add 10 packets. The first should be sent immediately since the buffers + // are clear. This will also trigger the probe to start. + EXPECT_CALL(packet_router, SendPacket).Times(AtLeast(1)); + pacer.CreateProbeCluster(kPacingDataRate * 2, 17); + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 10)); + time_controller.AdvanceTime(TimeDelta::Zero()); + ::testing::Mock::VerifyAndClearExpectations(&packet_router); + + // Advance time to 1ms before the coalescing window ends. Packets should be + // flying. + EXPECT_CALL(packet_router, SendPacket).Times(AtLeast(1)); + time_controller.AdvanceTime(kCoalescingWindow - TimeDelta::Millis(1)); + } + + TEST(TaskQueuePacedSenderTest, RespectedMinTimeBetweenStatsUpdates) { + const TimeDelta kCoalescingWindow = TimeDelta::Millis(5); + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + kCoalescingWindow); + const DataRate kPacingDataRate = DataRate::KilobitsPerSec(300); + pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); + + const TimeDelta kMinTimeBetweenStatsUpdates = TimeDelta::Millis(1); + + // Nothing inserted, no stats updates yet. + EXPECT_EQ(pacer.num_stats_updates_, 0u); + + // Insert one packet, stats should be updated. + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 1)); + time_controller.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(pacer.num_stats_updates_, 1u); + + // Advance time half of the min stats update interval, and trigger a + // refresh - stats should not be updated yet. + time_controller.AdvanceTime(kMinTimeBetweenStatsUpdates / 2); pacer.EnqueuePackets({}); time_controller.AdvanceTime(TimeDelta::Zero()); - EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); + EXPECT_EQ(pacer.num_stats_updates_, 1u); - // Advance time the rest of the way. - time_controller.AdvanceTime(kPacketPacingTime / 2); + // Advance time the next half, now stats update is triggered. + time_controller.AdvanceTime(kMinTimeBetweenStatsUpdates / 2); + pacer.EnqueuePackets({}); + time_controller.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(pacer.num_stats_updates_, 2u); } - // At this point, the pace queue is drained so there is no more intersting - // update to be made - but there is still as schduled task that should run - // |kMaxTimeBetweenStatsUpdates| after the first update. - time_controller.AdvanceTime(start_time + kMaxTimeBetweenStatsUpdates - - clock->CurrentTime()); - EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); - - // Advance time a significant time - don't expect any more calls as stats - // updating does not happen when queue is drained. - time_controller.AdvanceTime(TimeDelta::Millis(400)); - EXPECT_EQ(pacer.num_stats_updates_, num_expected_stats_updates); -} + TEST(TaskQueuePacedSenderTest, ThrottlesStatsUpdates) { + const TimeDelta kCoalescingWindow = TimeDelta::Millis(5); + GlobalSimulatedTimeController time_controller(Timestamp::Millis(1234)); + MockPacketRouter packet_router; + TaskQueuePacedSenderForTest pacer( + time_controller.GetClock(), &packet_router, + /*event_log=*/nullptr, + /*field_trials=*/nullptr, time_controller.GetTaskQueueFactory(), + kCoalescingWindow); + + // Set rates so one packet adds 10ms of buffer level. + const DataSize kPacketSize = DataSize::Bytes(kDefaultPacketSize); + const TimeDelta kPacketPacingTime = TimeDelta::Millis(10); + const DataRate kPacingDataRate = kPacketSize / kPacketPacingTime; + const TimeDelta kMinTimeBetweenStatsUpdates = TimeDelta::Millis(1); + const TimeDelta kMaxTimeBetweenStatsUpdates = TimeDelta::Millis(33); + + // Nothing inserted, no stats updates yet. + size_t num_expected_stats_updates = 0; + EXPECT_EQ(pacer.num_stats_updates_, num_expected_stats_updates); + pacer.SetPacingRates(kPacingDataRate, DataRate::Zero()); + time_controller.AdvanceTime(kMinTimeBetweenStatsUpdates); + // Updating pacing rates refreshes stats. + EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); + + // Record time when we insert first packet, this triggers the scheduled + // stats updating. + Clock* const clock = time_controller.GetClock(); + const Timestamp start_time = clock->CurrentTime(); + + while (clock->CurrentTime() - start_time <= + kMaxTimeBetweenStatsUpdates - kPacketPacingTime) { + // Enqueue packet, expect stats update. + pacer.EnqueuePackets(GeneratePackets(RtpPacketMediaType::kVideo, 1)); + time_controller.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); + + // Advance time to halfway through pacing time, expect another stats + // update. + time_controller.AdvanceTime(kPacketPacingTime / 2); + pacer.EnqueuePackets({}); + time_controller.AdvanceTime(TimeDelta::Zero()); + EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); + + // Advance time the rest of the way. + time_controller.AdvanceTime(kPacketPacingTime / 2); + } + + // At this point, the pace queue is drained so there is no more intersting + // update to be made - but there is still as schduled task that should run + // |kMaxTimeBetweenStatsUpdates| after the first update. + time_controller.AdvanceTime(start_time + kMaxTimeBetweenStatsUpdates - + clock->CurrentTime()); + EXPECT_EQ(pacer.num_stats_updates_, ++num_expected_stats_updates); + + // Advance time a significant time - don't expect any more calls as stats + // updating does not happen when queue is drained. + time_controller.AdvanceTime(TimeDelta::Millis(400)); + EXPECT_EQ(pacer.num_stats_updates_, num_expected_stats_updates); + } } // namespace test } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/remote_bitrate_estimator/BUILD.gn b/chromium/third_party/webrtc/modules/remote_bitrate_estimator/BUILD.gn index d7b0397ea5c..f5df205d7b6 100644 --- a/chromium/third_party/webrtc/modules/remote_bitrate_estimator/BUILD.gn +++ b/chromium/third_party/webrtc/modules/remote_bitrate_estimator/BUILD.gn @@ -59,6 +59,8 @@ rtc_library("remote_bitrate_estimator") { "../../system_wrappers", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -76,6 +78,8 @@ if (!build_with_chromium) { "../../rtc_base:rtc_base_approved", "../../test:rtp_test_utils", "../rtp_rtcp", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/flags:flag", "//third_party/abseil-cpp/absl/flags:parse", ] diff --git a/chromium/third_party/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy_unittest.cc b/chromium/third_party/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy_unittest.cc index f7e8ffc9fca..da995922d9d 100644 --- a/chromium/third_party/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy_unittest.cc +++ b/chromium/third_party/webrtc/modules/remote_bitrate_estimator/remote_estimator_proxy_unittest.cc @@ -65,9 +65,10 @@ std::vector<int64_t> TimestampsMs( class MockTransportFeedbackSender : public TransportFeedbackSenderInterface { public: - MOCK_METHOD1( - SendCombinedRtcpPacket, - bool(std::vector<std::unique_ptr<rtcp::RtcpPacket>> feedback_packets)); + MOCK_METHOD(bool, + SendCombinedRtcpPacket, + (std::vector<std::unique_ptr<rtcp::RtcpPacket>> feedback_packets), + (override)); }; class RemoteEstimatorProxyTest : public ::testing::Test { diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/BUILD.gn b/chromium/third_party/webrtc/modules/rtp_rtcp/BUILD.gn index 0ac6900e656..0446799fb7b 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/BUILD.gn +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/BUILD.gn @@ -118,6 +118,8 @@ rtc_library("rtp_rtcp_format") { "../../rtc_base/system:unused", "../../system_wrappers", "../video_coding:codec_globals_headers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", @@ -132,14 +134,18 @@ rtc_library("rtp_rtcp") { "include/flexfec_sender.h", "include/receive_statistics.h", "include/remote_ntp_time_estimator.h", - "include/rtp_rtcp.h", + "include/rtp_rtcp.h", # deprecated "include/ulpfec_receiver.h", "source/absolute_capture_time_receiver.cc", "source/absolute_capture_time_receiver.h", "source/absolute_capture_time_sender.cc", "source/absolute_capture_time_sender.h", + "source/active_decode_targets_helper.cc", + "source/active_decode_targets_helper.h", "source/create_video_rtp_depacketizer.cc", "source/create_video_rtp_depacketizer.h", + "source/deprecated/deprecated_rtp_sender_egress.cc", + "source/deprecated/deprecated_rtp_sender_egress.h", "source/dtmf_queue.cc", "source/dtmf_queue.h", "source/fec_private_tables_bursty.cc", @@ -186,6 +192,9 @@ rtc_library("rtp_rtcp") { "source/rtp_rtcp_config.h", "source/rtp_rtcp_impl.cc", "source/rtp_rtcp_impl.h", + "source/rtp_rtcp_impl2.cc", + "source/rtp_rtcp_impl2.h", + "source/rtp_rtcp_interface.h", "source/rtp_sender.cc", "source/rtp_sender.h", "source/rtp_sender_audio.cc", @@ -292,6 +301,8 @@ rtc_library("rtp_rtcp") { "../../system_wrappers:metrics", "../remote_bitrate_estimator", "../video_coding:codec_globals_headers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/container:inlined_vector", @@ -328,6 +339,8 @@ rtc_library("rtcp_transceiver") { "../../rtc_base/task_utils:repeating_task", "../../rtc_base/task_utils:to_queued_task", "../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", @@ -347,6 +360,8 @@ rtc_library("rtp_video_header") { "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", "../../modules/video_coding:codec_globals_headers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/types:optional", "//third_party/abseil-cpp/absl/types:variant", @@ -384,8 +399,8 @@ rtc_library("mock_rtp_rtcp") { "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "../../test:test_support", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } if (rtc_include_tests) { @@ -425,6 +440,7 @@ if (rtc_include_tests) { sources = [ "source/absolute_capture_time_receiver_unittest.cc", "source/absolute_capture_time_sender_unittest.cc", + "source/active_decode_targets_helper_unittest.cc", "source/byte_io_unittest.cc", "source/fec_private_tables_bursty_unittest.cc", "source/flexfec_header_reader_writer_unittest.cc", @@ -463,6 +479,7 @@ if (rtc_include_tests) { "source/rtcp_sender_unittest.cc", "source/rtcp_transceiver_impl_unittest.cc", "source/rtcp_transceiver_unittest.cc", + "source/rtp_dependency_descriptor_extension_unittest.cc", "source/rtp_fec_unittest.cc", "source/rtp_format_h264_unittest.cc", "source/rtp_format_unittest.cc", @@ -477,6 +494,7 @@ if (rtc_include_tests) { "source/rtp_packet_history_unittest.cc", "source/rtp_packet_unittest.cc", "source/rtp_packetizer_av1_unittest.cc", + "source/rtp_rtcp_impl2_unittest.cc", "source/rtp_rtcp_impl_unittest.cc", "source/rtp_sender_audio_unittest.cc", "source/rtp_sender_unittest.cc", @@ -541,6 +559,8 @@ if (rtc_include_tests) { "../../test:test_common", "../../test:test_support", "../video_coding:codec_globals_headers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/base:core_headers", "//third_party/abseil-cpp/absl/memory", diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_header_extension_map.h b/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_header_extension_map.h index 360a619f82f..ff2d34d60d9 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_header_extension_map.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_header_extension_map.h @@ -51,10 +51,6 @@ class RtpHeaderExtensionMap { return ids_[type]; } - // TODO(danilchap): Remove use of the functions below. - RTC_DEPRECATED int32_t Register(RTPExtensionType type, int id) { - return RegisterByType(id, type) ? 0 : -1; - } int32_t Deregister(RTPExtensionType type); void Deregister(absl::string_view uri); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp.h b/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp.h index f91f0d13a33..8663296eba0 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp.h @@ -12,456 +12,70 @@ #define MODULES_RTP_RTCP_INCLUDE_RTP_RTCP_H_ #include <memory> -#include <set> #include <string> -#include <utility> #include <vector> -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "api/frame_transformer_interface.h" -#include "api/scoped_refptr.h" -#include "api/transport/webrtc_key_value_config.h" -#include "api/video/video_bitrate_allocation.h" #include "modules/include/module.h" -#include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/report_block_data.h" -#include "modules/rtp_rtcp/include/rtp_packet_sender.h" -#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" -#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" -#include "modules/rtp_rtcp/source/video_fec_generator.h" -#include "rtc_base/constructor_magic.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "rtc_base/deprecation.h" namespace webrtc { -// Forward declarations. -class FrameEncryptorInterface; -class RateLimiter; -class ReceiveStatisticsProvider; -class RemoteBitrateEstimator; -class RtcEventLog; -class RTPSender; -class Transport; -class VideoBitrateAllocationObserver; - -namespace rtcp { -class TransportFeedback; -} - -class RtpRtcp : public Module, public RtcpFeedbackSenderInterface { +// DEPRECATED. Do not use. +class RtpRtcp : public Module, public RtpRtcpInterface { public: - struct Configuration { - Configuration(); - Configuration(Configuration&& rhs); - - // True for a audio version of the RTP/RTCP module object false will create - // a video version. - bool audio = false; - bool receiver_only = false; - - // The clock to use to read time. If nullptr then system clock will be used. - Clock* clock = nullptr; - - ReceiveStatisticsProvider* receive_statistics = nullptr; - - // Transport object that will be called when packets are ready to be sent - // out on the network. - Transport* outgoing_transport = nullptr; - - // Called when the receiver requests an intra frame. - RtcpIntraFrameObserver* intra_frame_callback = nullptr; - - // Called when the receiver sends a loss notification. - RtcpLossNotificationObserver* rtcp_loss_notification_observer = nullptr; - - // Called when we receive a changed estimate from the receiver of out - // stream. - RtcpBandwidthObserver* bandwidth_callback = nullptr; - - NetworkStateEstimateObserver* network_state_estimate_observer = nullptr; - TransportFeedbackObserver* transport_feedback_callback = nullptr; - VideoBitrateAllocationObserver* bitrate_allocation_observer = nullptr; - RtcpRttStats* rtt_stats = nullptr; - RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer = nullptr; - // Called on receipt of RTCP report block from remote side. - // TODO(bugs.webrtc.org/10678): Remove RtcpStatisticsCallback in - // favor of ReportBlockDataObserver. - // TODO(bugs.webrtc.org/10679): Consider whether we want to use - // only getters or only callbacks. If we decide on getters, the - // ReportBlockDataObserver should also be removed in favor of - // GetLatestReportBlockData(). - RtcpStatisticsCallback* rtcp_statistics_callback = nullptr; - RtcpCnameCallback* rtcp_cname_callback = nullptr; - ReportBlockDataObserver* report_block_data_observer = nullptr; - - // Estimates the bandwidth available for a set of streams from the same - // client. - RemoteBitrateEstimator* remote_bitrate_estimator = nullptr; - - // Spread any bursts of packets into smaller bursts to minimize packet loss. - RtpPacketSender* paced_sender = nullptr; - - // Generates FEC packets. - // TODO(sprang): Wire up to RtpSenderEgress. - VideoFecGenerator* fec_generator = nullptr; - - BitrateStatisticsObserver* send_bitrate_observer = nullptr; - SendSideDelayObserver* send_side_delay_observer = nullptr; - RtcEventLog* event_log = nullptr; - SendPacketObserver* send_packet_observer = nullptr; - RateLimiter* retransmission_rate_limiter = nullptr; - StreamDataCountersCallback* rtp_stats_callback = nullptr; - - int rtcp_report_interval_ms = 0; - - // Update network2 instead of pacer_exit field of video timing extension. - bool populate_network2_timestamp = false; - - rtc::scoped_refptr<FrameTransformerInterface> frame_transformer; - - // E2EE Custom Video Frame Encryption - FrameEncryptorInterface* frame_encryptor = nullptr; - // Require all outgoing frames to be encrypted with a FrameEncryptor. - bool require_frame_encryption = false; - - // Corresponds to extmap-allow-mixed in SDP negotiation. - bool extmap_allow_mixed = false; - - // If true, the RTP sender will always annotate outgoing packets with - // MID and RID header extensions, if provided and negotiated. - // If false, the RTP sender will stop sending MID and RID header extensions, - // when it knows that the receiver is ready to demux based on SSRC. This is - // done by RTCP RR acking. - bool always_send_mid_and_rid = false; - - // If set, field trials are read from |field_trials|, otherwise - // defaults to webrtc::FieldTrialBasedConfig. - const WebRtcKeyValueConfig* field_trials = nullptr; - - // SSRCs for media and retransmission, respectively. - // FlexFec SSRC is fetched from |flexfec_sender|. - uint32_t local_media_ssrc = 0; - absl::optional<uint32_t> rtx_send_ssrc; - - bool need_rtp_packet_infos = false; - - // If true, the RTP packet history will select RTX packets based on - // heuristics such as send time, retransmission count etc, in order to - // make padding potentially more useful. - // If false, the last packet will always be picked. This may reduce CPU - // overhead. - bool enable_rtx_padding_prioritization = true; - - private: - RTC_DISALLOW_COPY_AND_ASSIGN(Configuration); - }; - - // Creates an RTP/RTCP module object using provided |configuration|. - static std::unique_ptr<RtpRtcp> Create(const Configuration& configuration); + // Instantiates a deprecated version of the RtpRtcp module. + static std::unique_ptr<RtpRtcp> RTC_DEPRECATED + Create(const Configuration& configuration) { + return DEPRECATED_Create(configuration); + } - // ************************************************************************** - // Receiver functions - // ************************************************************************** + static std::unique_ptr<RtpRtcp> DEPRECATED_Create( + const Configuration& configuration); - virtual void IncomingRtcpPacket(const uint8_t* incoming_packet, - size_t incoming_packet_length) = 0; - - virtual void SetRemoteSSRC(uint32_t ssrc) = 0; - - // ************************************************************************** - // Sender - // ************************************************************************** - - // Sets the maximum size of an RTP packet, including RTP headers. - virtual void SetMaxRtpPacketSize(size_t size) = 0; + // (TMMBR) Temporary Max Media Bit Rate + RTC_DEPRECATED virtual bool TMMBR() const = 0; - // Returns max RTP packet size. Takes into account RTP headers and - // FEC/ULP/RED overhead (when FEC is enabled). - virtual size_t MaxRtpPacketSize() const = 0; + RTC_DEPRECATED virtual void SetTMMBRStatus(bool enable) = 0; - virtual void RegisterSendPayloadFrequency(int payload_type, - int payload_frequency) = 0; + // Returns -1 on failure else 0. + RTC_DEPRECATED virtual int32_t AddMixedCNAME(uint32_t ssrc, + const char* cname) = 0; - // Unregisters a send payload. - // |payload_type| - payload type of codec // Returns -1 on failure else 0. - virtual int32_t DeRegisterSendPayload(int8_t payload_type) = 0; + RTC_DEPRECATED virtual int32_t RemoveMixedCNAME(uint32_t ssrc) = 0; - virtual void SetExtmapAllowMixed(bool extmap_allow_mixed) = 0; + // Returns remote CName. + // Returns -1 on failure else 0. + RTC_DEPRECATED virtual int32_t RemoteCNAME( + uint32_t remote_ssrc, + char cname[RTCP_CNAME_SIZE]) const = 0; // (De)registers RTP header extension type and id. // Returns -1 on failure else 0. RTC_DEPRECATED virtual int32_t RegisterSendRtpHeaderExtension( RTPExtensionType type, uint8_t id) = 0; - // Register extension by uri, triggers CHECK on falure. - virtual void RegisterRtpHeaderExtension(absl::string_view uri, int id) = 0; - - virtual int32_t DeregisterSendRtpHeaderExtension(RTPExtensionType type) = 0; - virtual void DeregisterSendRtpHeaderExtension(absl::string_view uri) = 0; - - // Returns true if RTP module is send media, and any of the extensions - // required for bandwidth estimation is registered. - virtual bool SupportsPadding() const = 0; - // Same as SupportsPadding(), but additionally requires that - // SetRtxSendStatus() has been called with the kRtxRedundantPayloads option - // enabled. - virtual bool SupportsRtxPayloadPadding() const = 0; - - // Returns start timestamp. - virtual uint32_t StartTimestamp() const = 0; - - // Sets start timestamp. Start timestamp is set to a random value if this - // function is never called. - virtual void SetStartTimestamp(uint32_t timestamp) = 0; - - // Returns SequenceNumber. - virtual uint16_t SequenceNumber() const = 0; - - // Sets SequenceNumber, default is a random number. - virtual void SetSequenceNumber(uint16_t seq) = 0; - - virtual void SetRtpState(const RtpState& rtp_state) = 0; - virtual void SetRtxState(const RtpState& rtp_state) = 0; - virtual RtpState GetRtpState() const = 0; - virtual RtpState GetRtxState() const = 0; - - // Returns SSRC. - virtual uint32_t SSRC() const = 0; - - // Sets the value for sending in the RID (and Repaired) RTP header extension. - // RIDs are used to identify an RTP stream if SSRCs are not negotiated. - // If the RID and Repaired RID extensions are not registered, the RID will - // not be sent. - virtual void SetRid(const std::string& rid) = 0; - - // Sets the value for sending in the MID RTP header extension. - // The MID RTP header extension should be registered for this to do anything. - // Once set, this value can not be changed or removed. - virtual void SetMid(const std::string& mid) = 0; - - // Sets CSRC. - // |csrcs| - vector of CSRCs - virtual void SetCsrcs(const std::vector<uint32_t>& csrcs) = 0; - - // Turns on/off sending RTX (RFC 4588). The modes can be set as a combination - // of values of the enumerator RtxMode. - virtual void SetRtxSendStatus(int modes) = 0; - - // Returns status of sending RTX (RFC 4588). The returned value can be - // a combination of values of the enumerator RtxMode. - virtual int RtxSendStatus() const = 0; - - // Returns the SSRC used for RTX if set, otherwise a nullopt. - virtual absl::optional<uint32_t> RtxSsrc() const = 0; - - // Sets the payload type to use when sending RTX packets. Note that this - // doesn't enable RTX, only the payload type is set. - virtual void SetRtxSendPayloadType(int payload_type, - int associated_payload_type) = 0; - - // Returns the FlexFEC SSRC, if there is one. - virtual absl::optional<uint32_t> FlexfecSsrc() const = 0; - - // Sets sending status. Sends kRtcpByeCode when going from true to false. - // Returns -1 on failure else 0. - virtual int32_t SetSendingStatus(bool sending) = 0; - - // Returns current sending status. - virtual bool Sending() const = 0; - - // Starts/Stops media packets. On by default. - virtual void SetSendingMediaStatus(bool sending) = 0; - - // Returns current media sending status. - virtual bool SendingMedia() const = 0; - - // Returns whether audio is configured (i.e. Configuration::audio = true). - virtual bool IsAudioConfigured() const = 0; - - // Indicate that the packets sent by this module should be counted towards the - // bitrate estimate since the stream participates in the bitrate allocation. - virtual void SetAsPartOfAllocation(bool part_of_allocation) = 0; - - // TODO(sprang): Remove when all call sites have been moved to - // GetSendRates(). Fetches the current send bitrates in bits/s. - virtual void BitrateSent(uint32_t* total_rate, - uint32_t* video_rate, - uint32_t* fec_rate, - uint32_t* nack_rate) const = 0; - - // Returns bitrate sent (post-pacing) per packet type. - virtual RtpSendRates GetSendRates() const = 0; - - virtual RTPSender* RtpSender() = 0; - virtual const RTPSender* RtpSender() const = 0; - - // Record that a frame is about to be sent. Returns true on success, and false - // if the module isn't ready to send. - virtual bool OnSendingRtpFrame(uint32_t timestamp, - int64_t capture_time_ms, - int payload_type, - bool force_sender_report) = 0; - - // Try to send the provided packet. Returns true iff packet matches any of - // the SSRCs for this module (media/rtx/fec etc) and was forwarded to the - // transport. - virtual bool TrySendPacket(RtpPacketToSend* packet, - const PacedPacketInfo& pacing_info) = 0; - - virtual void OnPacketsAcknowledged( - rtc::ArrayView<const uint16_t> sequence_numbers) = 0; - - virtual std::vector<std::unique_ptr<RtpPacketToSend>> GeneratePadding( - size_t target_size_bytes) = 0; - - virtual std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( - rtc::ArrayView<const uint16_t> sequence_numbers) const = 0; - - // Returns an expected per packet overhead representing the main RTP header, - // any CSRCs, and the registered header extensions that are expected on all - // packets (i.e. disregarding things like abs capture time which is only - // populated on a subset of packets, but counting MID/RID type extensions - // when we expect to send them). - virtual size_t ExpectedPerPacketOverhead() const = 0; - - // ************************************************************************** - // RTCP - // ************************************************************************** - - // Returns RTCP status. - virtual RtcpMode RTCP() const = 0; - - // Sets RTCP status i.e on(compound or non-compound)/off. - // |method| - RTCP method to use. - virtual void SetRTCPStatus(RtcpMode method) = 0; - - // Sets RTCP CName (i.e unique identifier). - // Returns -1 on failure else 0. - virtual int32_t SetCNAME(const char* cname) = 0; - - // Returns remote CName. - // Returns -1 on failure else 0. - virtual int32_t RemoteCNAME(uint32_t remote_ssrc, - char cname[RTCP_CNAME_SIZE]) const = 0; - - // Returns remote NTP. - // Returns -1 on failure else 0. - virtual int32_t RemoteNTP(uint32_t* received_ntp_secs, - uint32_t* received_ntp_frac, - uint32_t* rtcp_arrival_time_secs, - uint32_t* rtcp_arrival_time_frac, - uint32_t* rtcp_timestamp) const = 0; - - // Returns -1 on failure else 0. - virtual int32_t AddMixedCNAME(uint32_t ssrc, const char* cname) = 0; - - // Returns -1 on failure else 0. - virtual int32_t RemoveMixedCNAME(uint32_t ssrc) = 0; - - // Returns current RTT (round-trip time) estimate. - // Returns -1 on failure else 0. - virtual int32_t RTT(uint32_t remote_ssrc, - int64_t* rtt, - int64_t* avg_rtt, - int64_t* min_rtt, - int64_t* max_rtt) const = 0; - - // Returns the estimated RTT, with fallback to a default value. - virtual int64_t ExpectedRetransmissionTimeMs() const = 0; - - // Forces a send of a RTCP packet. Periodic SR and RR are triggered via the - // process function. - // Returns -1 on failure else 0. - virtual int32_t SendRTCP(RTCPPacketType rtcp_packet_type) = 0; - - // Returns statistics of the amount of data sent. - // Returns -1 on failure else 0. - virtual int32_t DataCountersRTP(size_t* bytes_sent, - uint32_t* packets_sent) const = 0; - - // Returns send statistics for the RTP and RTX stream. - virtual void GetSendStreamDataCounters( - StreamDataCounters* rtp_counters, - StreamDataCounters* rtx_counters) const = 0; - - // Returns received RTCP report block. - // Returns -1 on failure else 0. - // TODO(https://crbug.com/webrtc/10678): Remove this in favor of - // GetLatestReportBlockData(). - virtual int32_t RemoteRTCPStat( - std::vector<RTCPReportBlock>* receive_blocks) const = 0; - // A snapshot of Report Blocks with additional data of interest to statistics. - // Within this list, the sender-source SSRC pair is unique and per-pair the - // ReportBlockData represents the latest Report Block that was received for - // that pair. - virtual std::vector<ReportBlockData> GetLatestReportBlockData() const = 0; // (APP) Sets application specific data. // Returns -1 on failure else 0. - virtual int32_t SetRTCPApplicationSpecificData(uint8_t sub_type, - uint32_t name, - const uint8_t* data, - uint16_t length) = 0; - // (XR) Sets Receiver Reference Time Report (RTTR) status. - virtual void SetRtcpXrRrtrStatus(bool enable) = 0; - - // Returns current Receiver Reference Time Report (RTTR) status. - virtual bool RtcpXrRrtrStatus() const = 0; + RTC_DEPRECATED virtual int32_t SetRTCPApplicationSpecificData( + uint8_t sub_type, + uint32_t name, + const uint8_t* data, + uint16_t length) = 0; - // (REMB) Receiver Estimated Max Bitrate. - // Schedules sending REMB on next and following sender/receiver reports. - void SetRemb(int64_t bitrate_bps, std::vector<uint32_t> ssrcs) override = 0; - // Stops sending REMB on next and following sender/receiver reports. - void UnsetRemb() override = 0; - - // (TMMBR) Temporary Max Media Bit Rate - virtual bool TMMBR() const = 0; - - virtual void SetTMMBRStatus(bool enable) = 0; - - // (NACK) - - // Sends a Negative acknowledgement packet. + // Returns statistics of the amount of data sent. // Returns -1 on failure else 0. - // TODO(philipel): Deprecate this and start using SendNack instead, mostly - // because we want a function that actually send NACK for the specified - // packets. - virtual int32_t SendNACK(const uint16_t* nack_list, uint16_t size) = 0; - - // Sends NACK for the packets specified. - // Note: This assumes the caller keeps track of timing and doesn't rely on - // the RTP module to do this. - virtual void SendNack(const std::vector<uint16_t>& sequence_numbers) = 0; - - // Store the sent packets, needed to answer to a Negative acknowledgment - // requests. - virtual void SetStorePacketsStatus(bool enable, uint16_t numberToStore) = 0; - - // Returns true if the module is configured to store packets. - virtual bool StorePackets() const = 0; - - virtual void SetVideoBitrateAllocation( - const VideoBitrateAllocation& bitrate) = 0; - - // ************************************************************************** - // Video - // ************************************************************************** + RTC_DEPRECATED virtual int32_t DataCountersRTP( + size_t* bytes_sent, + uint32_t* packets_sent) const = 0; // Requests new key frame. // using PLI, https://tools.ietf.org/html/rfc4585#section-6.3.1.1 void SendPictureLossIndication() { SendRTCP(kRtcpPli); } // using FIR, https://tools.ietf.org/html/rfc5104#section-4.3.1.2 void SendFullIntraRequest() { SendRTCP(kRtcpFir); } - - // Sends a LossNotification RTCP message. - // Returns -1 on failure else 0. - virtual int32_t SendLossNotification(uint16_t last_decoded_seq_num, - uint16_t last_received_seq_num, - bool decodability_flag, - bool buffering_allowed) = 0; }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h index 049ff5c506d..1b72236bbb3 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -65,7 +65,6 @@ enum RTPExtensionType : int { kRtpExtensionPlayoutDelay, kRtpExtensionVideoContentType, kRtpExtensionVideoTiming, - kRtpExtensionFrameMarking, kRtpExtensionRtpStreamId, kRtpExtensionRepairedRtpStreamId, kRtpExtensionMid, @@ -91,7 +90,6 @@ enum RTCPPacketType : uint32_t { kRtcpTmmbr = 0x0100, kRtcpTmmbn = 0x0200, kRtcpSrReq = 0x0400, - kRtcpApp = 0x1000, kRtcpLossNotification = 0x2000, kRtcpRemb = 0x10000, kRtcpTransmissionTimeOffset = 0x20000, diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtcp_rtt_stats.h b/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtcp_rtt_stats.h index 5b1585fa0fb..e9a7d526916 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtcp_rtt_stats.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtcp_rtt_stats.h @@ -19,7 +19,7 @@ namespace webrtc { class MockRtcpRttStats : public RtcpRttStats { public: MOCK_METHOD(void, OnRttUpdate, (int64_t rtt), (override)); - MOCK_METHOD(int64_t, LastProcessedRtt, (), (const override)); + MOCK_METHOD(int64_t, LastProcessedRtt, (), (const, override)); }; } // namespace webrtc #endif // MODULES_RTP_RTCP_MOCKS_MOCK_RTCP_RTT_STATS_H_ diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h b/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h index 5a333fe8470..08b38eee7b6 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h @@ -20,14 +20,14 @@ #include "absl/types/optional.h" #include "api/video/video_bitrate_allocation.h" #include "modules/include/module.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "test/gmock.h" namespace webrtc { -class MockRtpRtcp : public RtpRtcp { +class MockRtpRtcpInterface : public RtpRtcpInterface { public: MOCK_METHOD(void, IncomingRtcpPacket, @@ -35,7 +35,7 @@ class MockRtpRtcp : public RtpRtcp { (override)); MOCK_METHOD(void, SetRemoteSSRC, (uint32_t ssrc), (override)); MOCK_METHOD(void, SetMaxRtpPacketSize, (size_t size), (override)); - MOCK_METHOD(size_t, MaxRtpPacketSize, (), (const override)); + MOCK_METHOD(size_t, MaxRtpPacketSize, (), (const, override)); MOCK_METHOD(void, RegisterSendPayloadFrequency, (int payload_type, int frequency), @@ -45,10 +45,6 @@ class MockRtpRtcp : public RtpRtcp { (int8_t payload_type), (override)); MOCK_METHOD(void, SetExtmapAllowMixed, (bool extmap_allow_mixed), (override)); - MOCK_METHOD(int32_t, - RegisterSendRtpHeaderExtension, - (RTPExtensionType type, uint8_t id), - (override)); MOCK_METHOD(void, RegisterRtpHeaderExtension, (absl::string_view uri, int id), @@ -61,32 +57,30 @@ class MockRtpRtcp : public RtpRtcp { DeregisterSendRtpHeaderExtension, (absl::string_view uri), (override)); - MOCK_METHOD(bool, SupportsPadding, (), (const override)); - MOCK_METHOD(bool, SupportsRtxPayloadPadding, (), (const override)); - MOCK_METHOD(uint32_t, StartTimestamp, (), (const override)); + MOCK_METHOD(bool, SupportsPadding, (), (const, override)); + MOCK_METHOD(bool, SupportsRtxPayloadPadding, (), (const, override)); + MOCK_METHOD(uint32_t, StartTimestamp, (), (const, override)); MOCK_METHOD(void, SetStartTimestamp, (uint32_t timestamp), (override)); - MOCK_METHOD(uint16_t, SequenceNumber, (), (const override)); + MOCK_METHOD(uint16_t, SequenceNumber, (), (const, override)); MOCK_METHOD(void, SetSequenceNumber, (uint16_t seq), (override)); MOCK_METHOD(void, SetRtpState, (const RtpState& rtp_state), (override)); MOCK_METHOD(void, SetRtxState, (const RtpState& rtp_state), (override)); - MOCK_METHOD(RtpState, GetRtpState, (), (const override)); - MOCK_METHOD(RtpState, GetRtxState, (), (const override)); - MOCK_METHOD(uint32_t, SSRC, (), (const override)); + MOCK_METHOD(RtpState, GetRtpState, (), (const, override)); + MOCK_METHOD(RtpState, GetRtxState, (), (const, override)); + MOCK_METHOD(uint32_t, SSRC, (), (const, override)); MOCK_METHOD(void, SetRid, (const std::string& rid), (override)); MOCK_METHOD(void, SetMid, (const std::string& mid), (override)); - MOCK_METHOD(int32_t, CSRCs, (uint32_t csrcs[kRtpCsrcSize]), (const override)); MOCK_METHOD(void, SetCsrcs, (const std::vector<uint32_t>& csrcs), (override)); MOCK_METHOD(void, SetRtxSendStatus, (int modes), (override)); - MOCK_METHOD(int, RtxSendStatus, (), (const override)); - MOCK_METHOD(absl::optional<uint32_t>, RtxSsrc, (), (const override)); + MOCK_METHOD(int, RtxSendStatus, (), (const, override)); + MOCK_METHOD(absl::optional<uint32_t>, RtxSsrc, (), (const, override)); MOCK_METHOD(void, SetRtxSendPayloadType, (int, int), (override)); - MOCK_METHOD(absl::optional<uint32_t>, FlexfecSsrc, (), (const override)); - MOCK_METHOD((std::pair<int, int>), RtxSendPayloadType, (), (const override)); + MOCK_METHOD(absl::optional<uint32_t>, FlexfecSsrc, (), (const, override)); MOCK_METHOD(int32_t, SetSendingStatus, (bool sending), (override)); - MOCK_METHOD(bool, Sending, (), (const override)); + MOCK_METHOD(bool, Sending, (), (const, override)); MOCK_METHOD(void, SetSendingMediaStatus, (bool sending), (override)); - MOCK_METHOD(bool, SendingMedia, (), (const override)); - MOCK_METHOD(bool, IsAudioConfigured, (), (const override)); + MOCK_METHOD(bool, SendingMedia, (), (const, override)); + MOCK_METHOD(bool, IsAudioConfigured, (), (const, override)); MOCK_METHOD(void, SetAsPartOfAllocation, (bool), (override)); MOCK_METHOD(void, BitrateSent, @@ -94,12 +88,8 @@ class MockRtpRtcp : public RtpRtcp { uint32_t* video_rate, uint32_t* fec_rate, uint32_t* nack_rate), - (const override)); - MOCK_METHOD(RtpSendRates, GetSendRates, (), (const override)); - MOCK_METHOD(int, - EstimatedReceiveBandwidth, - (uint32_t * available_bandwidth), - (const override)); + (const, override)); + MOCK_METHOD(RtpSendRates, GetSendRates, (), (const, override)); MOCK_METHOD(bool, OnSendingRtpFrame, (uint32_t, int64_t, int, bool), @@ -119,31 +109,22 @@ class MockRtpRtcp : public RtpRtcp { MOCK_METHOD(std::vector<RtpSequenceNumberMap::Info>, GetSentRtpPacketInfos, (rtc::ArrayView<const uint16_t> sequence_numbers), - (const override)); - MOCK_METHOD(size_t, ExpectedPerPacketOverhead, (), (const override)); - MOCK_METHOD(RtcpMode, RTCP, (), (const override)); + (const, override)); + MOCK_METHOD(size_t, ExpectedPerPacketOverhead, (), (const, override)); + MOCK_METHOD(RtcpMode, RTCP, (), (const, override)); MOCK_METHOD(void, SetRTCPStatus, (RtcpMode method), (override)); MOCK_METHOD(int32_t, SetCNAME, (const char cname[RTCP_CNAME_SIZE]), (override)); MOCK_METHOD(int32_t, - RemoteCNAME, - (uint32_t remote_ssrc, char cname[RTCP_CNAME_SIZE]), - (const override)); - MOCK_METHOD(int32_t, RemoteNTP, (uint32_t * received_ntp_secs, uint32_t* received_ntp_frac, uint32_t* rtcp_arrival_time_secs, uint32_t* rtcp_arrival_time_frac, uint32_t* rtcp_timestamp), - (const override)); - MOCK_METHOD(int32_t, - AddMixedCNAME, - (uint32_t ssrc, const char cname[RTCP_CNAME_SIZE]), - (override)); - MOCK_METHOD(int32_t, RemoveMixedCNAME, (uint32_t ssrc), (override)); + (const, override)); MOCK_METHOD(int32_t, RTT, (uint32_t remote_ssrc, @@ -151,39 +132,28 @@ class MockRtpRtcp : public RtpRtcp { int64_t* avg_rtt, int64_t* min_rtt, int64_t* max_rtt), - (const override)); - MOCK_METHOD(int64_t, ExpectedRetransmissionTimeMs, (), (const override)); + (const, override)); + MOCK_METHOD(int64_t, ExpectedRetransmissionTimeMs, (), (const, override)); MOCK_METHOD(int32_t, SendRTCP, (RTCPPacketType packet_type), (override)); - MOCK_METHOD(int32_t, - DataCountersRTP, - (size_t * bytes_sent, uint32_t* packets_sent), - (const override)); MOCK_METHOD(void, GetSendStreamDataCounters, (StreamDataCounters*, StreamDataCounters*), - (const override)); + (const, override)); MOCK_METHOD(int32_t, RemoteRTCPStat, (std::vector<RTCPReportBlock> * receive_blocks), - (const override)); + (const, override)); MOCK_METHOD(std::vector<ReportBlockData>, GetLatestReportBlockData, (), - (const override)); - MOCK_METHOD( - int32_t, - SetRTCPApplicationSpecificData, - (uint8_t sub_type, uint32_t name, const uint8_t* data, uint16_t length), - (override)); + (const, override)); MOCK_METHOD(void, SetRtcpXrRrtrStatus, (bool enable), (override)); - MOCK_METHOD(bool, RtcpXrRrtrStatus, (), (const override)); + MOCK_METHOD(bool, RtcpXrRrtrStatus, (), (const, override)); MOCK_METHOD(void, SetRemb, (int64_t bitrate, std::vector<uint32_t> ssrcs), (override)); MOCK_METHOD(void, UnsetRemb, (), (override)); - MOCK_METHOD(bool, TMMBR, (), (const override)); - MOCK_METHOD(void, SetTMMBRStatus, (bool enable), (override)); MOCK_METHOD(int32_t, SendNACK, (const uint16_t* nack_list, uint16_t size), @@ -196,7 +166,7 @@ class MockRtpRtcp : public RtpRtcp { SetStorePacketsStatus, (bool enable, uint16_t number_to_store), (override)); - MOCK_METHOD(bool, StorePackets, (), (const override)); + MOCK_METHOD(bool, StorePackets, (), (const, override)); MOCK_METHOD(void, SendCombinedRtcpPacket, (std::vector<std::unique_ptr<rtcp::RtcpPacket>> rtcp_packets), @@ -208,20 +178,12 @@ class MockRtpRtcp : public RtpRtcp { bool decodability_flag, bool buffering_allowed), (override)); - MOCK_METHOD(void, Process, (), (override)); MOCK_METHOD(void, SetVideoBitrateAllocation, (const VideoBitrateAllocation&), (override)); MOCK_METHOD(RTPSender*, RtpSender, (), (override)); - MOCK_METHOD(const RTPSender*, RtpSender, (), (const override)); - - private: - // Mocking this method is currently not required and having a default - // implementation like - // MOCK_METHOD(int64_t, TimeUntilNextProcess, (), (override)) - // can be dangerous since it can cause a tight loop on a process thread. - int64_t TimeUntilNextProcess() override { return 0xffffffff; } + MOCK_METHOD(const RTPSender*, RtpSender, (), (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper.cc new file mode 100644 index 00000000000..a14426e1445 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper.cc @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/active_decode_targets_helper.h" + +#include <stdint.h> + +#include "api/array_view.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +// Returns mask of ids of chains previous frame is part of. +// Assumes for each chain frames are seen in order and no frame on any chain is +// missing. That assumptions allows a simple detection when previous frame is +// part of a chain. +std::bitset<32> LastSendOnChain(int frame_diff, + rtc::ArrayView<const int> chain_diffs) { + std::bitset<32> bitmask = 0; + for (size_t i = 0; i < chain_diffs.size(); ++i) { + if (frame_diff == chain_diffs[i]) { + bitmask.set(i); + } + } + return bitmask; +} + +// Returns bitmask with first `num` bits set to 1. +std::bitset<32> AllActive(size_t num) { + RTC_DCHECK_LE(num, 32); + return (~uint32_t{0}) >> (32 - num); +} + +// Returns bitmask of chains that protect at least one active decode target. +std::bitset<32> ActiveChains( + rtc::ArrayView<const int> decode_target_protected_by_chain, + int num_chains, + std::bitset<32> active_decode_targets) { + std::bitset<32> active_chains = 0; + for (size_t dt = 0; dt < decode_target_protected_by_chain.size(); ++dt) { + if (dt < active_decode_targets.size() && !active_decode_targets[dt]) { + continue; + } + // chain_idx == num_chains is valid and means the decode target is + // not protected by any chain. + int chain_idx = decode_target_protected_by_chain[dt]; + if (chain_idx < num_chains) { + active_chains.set(chain_idx); + } + } + return active_chains; +} + +} // namespace + +void ActiveDecodeTargetsHelper::OnFrame( + rtc::ArrayView<const int> decode_target_protected_by_chain, + std::bitset<32> active_decode_targets, + bool is_keyframe, + int64_t frame_id, + rtc::ArrayView<const int> chain_diffs) { + const int num_chains = chain_diffs.size(); + if (num_chains == 0) { + // Avoid printing the warning + // when already printed the warning for the same active decode targets, or + // when active_decode_targets are not changed from it's default value of + // all are active, including non-existent decode targets. + if (last_active_decode_targets_ != active_decode_targets && + !active_decode_targets.all()) { + RTC_LOG(LS_WARNING) << "No chains are configured, but some decode " + "targets might be inactive. Unsupported."; + } + last_active_decode_targets_ = active_decode_targets; + return; + } + const size_t num_decode_targets = decode_target_protected_by_chain.size(); + RTC_DCHECK_GT(num_decode_targets, 0); + std::bitset<32> all_decode_targets = AllActive(num_decode_targets); + // Default value for active_decode_targets is 'all are active', i.e. all bits + // are set. Default value is set before number of decode targets is known. + // It is up to this helper to make the value cleaner and unset unused bits. + active_decode_targets &= all_decode_targets; + + if (is_keyframe) { + // Key frame resets the state. + last_active_decode_targets_ = all_decode_targets; + unsent_on_chain_.reset(); + } else { + // Update state assuming previous frame was sent. + unsent_on_chain_ &= + ~LastSendOnChain(frame_id - last_frame_id_, chain_diffs); + } + // Save for the next call to OnFrame. + // Though usually `frame_id == last_frame_id_ + 1`, it might not be so when + // frame id space is shared by several simulcast rtp streams. + last_frame_id_ = frame_id; + + if (active_decode_targets == last_active_decode_targets_) { + return; + } + last_active_decode_targets_ = active_decode_targets; + + // Frames that are part of inactive chains might not be produced by the + // encoder. Thus stop sending `active_decode_target` bitmask when it is sent + // on all active chains rather than on all chains. + unsent_on_chain_ = ActiveChains(decode_target_protected_by_chain, num_chains, + active_decode_targets); + if (unsent_on_chain_.none()) { + // Active decode targets are not protected by any chains. To be on the + // safe side always send the active_decode_targets_bitmask from now on. + RTC_LOG(LS_WARNING) + << "Active decode targets protected by no chains. (In)active decode " + "targets information will be send overreliably."; + unsent_on_chain_.set(1); + } +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper.h new file mode 100644 index 00000000000..b51144d9cb1 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_ACTIVE_DECODE_TARGETS_HELPER_H_ +#define MODULES_RTP_RTCP_SOURCE_ACTIVE_DECODE_TARGETS_HELPER_H_ + +#include <stdint.h> + +#include <bitset> + +#include "absl/types/optional.h" +#include "api/array_view.h" + +namespace webrtc { + +// Helper class that decides when active_decode_target_bitmask should be written +// into the dependency descriptor rtp header extension. +// See: https://aomediacodec.github.io/av1-rtp-spec/#a44-switching +// This class is thread-compatible +class ActiveDecodeTargetsHelper { + public: + ActiveDecodeTargetsHelper() = default; + ActiveDecodeTargetsHelper(const ActiveDecodeTargetsHelper&) = delete; + ActiveDecodeTargetsHelper& operator=(const ActiveDecodeTargetsHelper&) = + delete; + ~ActiveDecodeTargetsHelper() = default; + + // Decides if active decode target bitmask should be attached to the frame + // that is about to be sent. + void OnFrame(rtc::ArrayView<const int> decode_target_protected_by_chain, + std::bitset<32> active_decode_targets, + bool is_keyframe, + int64_t frame_id, + rtc::ArrayView<const int> chain_diffs); + + // Returns active decode target to attach to the dependency descriptor. + absl::optional<uint32_t> ActiveDecodeTargetsBitmask() const { + if (unsent_on_chain_.none()) + return absl::nullopt; + return last_active_decode_targets_.to_ulong(); + } + + private: + // `unsent_on_chain_[i]` indicates last active decode + // target bitmask wasn't attached to a packet on the chain with id `i`. + std::bitset<32> unsent_on_chain_ = 0; + std::bitset<32> last_active_decode_targets_ = 0; + int64_t last_frame_id_ = 0; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_ACTIVE_DECODE_TARGETS_HELPER_H_ diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc new file mode 100644 index 00000000000..651ab22e546 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/active_decode_targets_helper_unittest.cc @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/active_decode_targets_helper.h" + +#include <vector> + +#include "absl/types/optional.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { +constexpr std::bitset<32> kAll = ~uint32_t{0}; +} // namespace + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptOnKeyFrameWhenAllDecodeTargetsAreActive) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptOnKeyFrameWhenAllDecodeTargetsAreActiveAfterDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/3, chain_diffs_key); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsBitmaskOnKeyFrameWhenSomeDecodeTargetsAreInactive) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsBitmaskOnKeyFrameWhenSomeDecodeTargetsAreInactiveAfterDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/3, chain_diffs_key); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptWhenActiveDecodeTargetsAreUnused) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsNulloptOnDeltaFrameAfterSentOnKeyFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsNewBitmaskOnDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b11, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b01u); +} + +TEST(ActiveDecodeTargetsHelperTest, + ReturnsBitmaskWhenAllDecodeTargetsReactivatedOnDeltaFrame) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 0}; + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/true, /*frame_id=*/1, chain_diffs_key); + ASSERT_NE(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b01, + /*is_keyframe=*/false, /*frame_id=*/2, chain_diffs_delta); + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + // Reactive all the decode targets + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/false, /*frame_id=*/3, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b11u); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsNulloptAfterSentOnAllActiveChains) { + // Active decode targets (0 and 1) are protected by chains 1 and 2. + const std::bitset<32> kSome = 0b011; + constexpr int kDecodeTargetProtectedByChain[] = {2, 1, 0}; + + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0, 0, 0}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b111, + /*is_keyframe=*/true, + /*frame_id=*/0, chain_diffs_key); + ASSERT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + int chain_diffs_delta1[] = {1, 1, 1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kSome, + /*is_keyframe=*/false, + /*frame_id=*/1, chain_diffs_delta1); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b011u); + + int chain_diffs_delta2[] = {2, 2, 1}; // Previous frame was part of chain#2 + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kSome, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta2); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b011u); + + // active_decode_targets_bitmask was send on chains 1 and 2. It was never sent + // on chain 0, but chain 0 only protects inactive decode target#2 + int chain_diffs_delta3[] = {3, 1, 2}; // Previous frame was part of chain#1 + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/kSome, + /*is_keyframe=*/false, + /*frame_id=*/3, chain_diffs_delta3); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsBitmaskWhenChanged) { + constexpr int kDecodeTargetProtectedByChain[] = {0, 1, 1}; + + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0, 0}; + helper.OnFrame(kDecodeTargetProtectedByChain, /*active_decode_targets=*/0b111, + /*is_keyframe=*/true, + /*frame_id=*/0, chain_diffs_key); + int chain_diffs_delta1[] = {1, 1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b011, + /*is_keyframe=*/false, + /*frame_id=*/1, chain_diffs_delta1); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b011u); + + int chain_diffs_delta2[] = {1, 2}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b101, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta2); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b101u); + + // active_decode_target_bitmask was send on chain0, but it was an old one. + int chain_diffs_delta3[] = {2, 1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b101, + /*is_keyframe=*/false, + /*frame_id=*/3, chain_diffs_delta3); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b101u); +} + +TEST(ActiveDecodeTargetsHelperTest, ReturnsNulloptWhenChainsAreNotUsed) { + const rtc::ArrayView<const int> kDecodeTargetProtectedByChain; + const rtc::ArrayView<const int> kNoChainDiffs; + + ActiveDecodeTargetsHelper helper; + helper.OnFrame(kDecodeTargetProtectedByChain, /*active_decode_targets=*/kAll, + /*is_keyframe=*/true, + /*frame_id=*/0, kNoChainDiffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b101, + /*is_keyframe=*/false, + /*frame_id=*/1, kNoChainDiffs); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); +} + +TEST(ActiveDecodeTargetsHelperTest, + KeepReturningBitmaskWhenAllChainsAreInactive) { + // Two decode targets, but single chain. + // 2nd decode target is not protected by any chain. + constexpr int kDecodeTargetProtectedByChain[] = {0, 1}; + + ActiveDecodeTargetsHelper helper; + int chain_diffs_key[] = {0}; + helper.OnFrame(kDecodeTargetProtectedByChain, /*active_decode_targets=*/0b10, + /*is_keyframe=*/true, + /*frame_id=*/0, chain_diffs_key); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b10u); + + // Even though previous frame is part of the only chain, that inactive chain + // doesn't provide guaranted delivery. + int chain_diffs_delta[] = {1}; + helper.OnFrame(kDecodeTargetProtectedByChain, + /*active_decode_targets=*/0b10, + /*is_keyframe=*/false, + /*frame_id=*/1, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), 0b10u); +} + +TEST(ActiveDecodeTargetsHelperTest, Supports32DecodeTargets) { + std::bitset<32> some; + std::vector<int> decode_target_protected_by_chain(32); + for (int i = 0; i < 32; ++i) { + decode_target_protected_by_chain[i] = i; + some[i] = i % 2 == 0; + } + + ActiveDecodeTargetsHelper helper; + std::vector<int> chain_diffs_key(32, 0); + helper.OnFrame(decode_target_protected_by_chain, + /*active_decode_targets=*/some, + /*is_keyframe=*/true, + /*frame_id=*/1, chain_diffs_key); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), some.to_ulong()); + std::vector<int> chain_diffs_delta(32, 1); + helper.OnFrame(decode_target_protected_by_chain, + /*active_decode_targets=*/some, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), absl::nullopt); + helper.OnFrame(decode_target_protected_by_chain, + /*active_decode_targets=*/kAll, + /*is_keyframe=*/false, + /*frame_id=*/2, chain_diffs_delta); + EXPECT_EQ(helper.ActiveDecodeTargetsBitmask(), kAll.to_ulong()); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.cc new file mode 100644 index 00000000000..b13c85a1d78 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.cc @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.h" + +#include <limits> +#include <memory> +#include <utility> + +#include "absl/strings/match.h" +#include "api/transport/field_trial_based_config.h" +#include "logging/rtc_event_log/events/rtc_event_rtp_packet_outgoing.h" +#include "modules/remote_bitrate_estimator/test/bwe_test_logging.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { +constexpr uint32_t kTimestampTicksPerMs = 90; +constexpr int kSendSideDelayWindowMs = 1000; +constexpr int kBitrateStatisticsWindowMs = 1000; +constexpr size_t kRtpSequenceNumberMapMaxEntries = 1 << 13; + +bool IsEnabled(absl::string_view name, + const WebRtcKeyValueConfig* field_trials) { + FieldTrialBasedConfig default_trials; + auto& trials = field_trials ? *field_trials : default_trials; + return absl::StartsWith(trials.Lookup(name), "Enabled"); +} +} // namespace + +DEPRECATED_RtpSenderEgress::NonPacedPacketSender::NonPacedPacketSender( + DEPRECATED_RtpSenderEgress* sender) + : transport_sequence_number_(0), sender_(sender) {} +DEPRECATED_RtpSenderEgress::NonPacedPacketSender::~NonPacedPacketSender() = + default; + +void DEPRECATED_RtpSenderEgress::NonPacedPacketSender::EnqueuePackets( + std::vector<std::unique_ptr<RtpPacketToSend>> packets) { + for (auto& packet : packets) { + if (!packet->SetExtension<TransportSequenceNumber>( + ++transport_sequence_number_)) { + --transport_sequence_number_; + } + packet->ReserveExtension<TransmissionOffset>(); + packet->ReserveExtension<AbsoluteSendTime>(); + sender_->SendPacket(packet.get(), PacedPacketInfo()); + } +} + +DEPRECATED_RtpSenderEgress::DEPRECATED_RtpSenderEgress( + const RtpRtcpInterface::Configuration& config, + RtpPacketHistory* packet_history) + : ssrc_(config.local_media_ssrc), + rtx_ssrc_(config.rtx_send_ssrc), + flexfec_ssrc_(config.fec_generator ? config.fec_generator->FecSsrc() + : absl::nullopt), + populate_network2_timestamp_(config.populate_network2_timestamp), + send_side_bwe_with_overhead_( + IsEnabled("WebRTC-SendSideBwe-WithOverhead", config.field_trials)), + clock_(config.clock), + packet_history_(packet_history), + transport_(config.outgoing_transport), + event_log_(config.event_log), + is_audio_(config.audio), + need_rtp_packet_infos_(config.need_rtp_packet_infos), + transport_feedback_observer_(config.transport_feedback_callback), + send_side_delay_observer_(config.send_side_delay_observer), + send_packet_observer_(config.send_packet_observer), + rtp_stats_callback_(config.rtp_stats_callback), + bitrate_callback_(config.send_bitrate_observer), + media_has_been_sent_(false), + force_part_of_allocation_(false), + timestamp_offset_(0), + max_delay_it_(send_delays_.end()), + sum_delays_ms_(0), + total_packet_send_delay_ms_(0), + send_rates_(kNumMediaTypes, + {kBitrateStatisticsWindowMs, RateStatistics::kBpsScale}), + rtp_sequence_number_map_(need_rtp_packet_infos_ + ? std::make_unique<RtpSequenceNumberMap>( + kRtpSequenceNumberMapMaxEntries) + : nullptr) {} + +void DEPRECATED_RtpSenderEgress::SendPacket( + RtpPacketToSend* packet, + const PacedPacketInfo& pacing_info) { + RTC_DCHECK(packet); + + const uint32_t packet_ssrc = packet->Ssrc(); + RTC_DCHECK(packet->packet_type().has_value()); + RTC_DCHECK(HasCorrectSsrc(*packet)); + int64_t now_ms = clock_->TimeInMilliseconds(); + + if (is_audio_) { +#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "AudioTotBitrate_kbps", now_ms, + GetSendRates().Sum().kbps(), packet_ssrc); + BWE_TEST_LOGGING_PLOT_WITH_SSRC( + 1, "AudioNackBitrate_kbps", now_ms, + GetSendRates()[RtpPacketMediaType::kRetransmission].kbps(), + packet_ssrc); +#endif + } else { +#if BWE_TEST_LOGGING_COMPILE_TIME_ENABLE + BWE_TEST_LOGGING_PLOT_WITH_SSRC(1, "VideoTotBitrate_kbps", now_ms, + GetSendRates().Sum().kbps(), packet_ssrc); + BWE_TEST_LOGGING_PLOT_WITH_SSRC( + 1, "VideoNackBitrate_kbps", now_ms, + GetSendRates()[RtpPacketMediaType::kRetransmission].kbps(), + packet_ssrc); +#endif + } + + PacketOptions options; + { + rtc::CritScope lock(&lock_); + options.included_in_allocation = force_part_of_allocation_; + + if (need_rtp_packet_infos_ && + packet->packet_type() == RtpPacketToSend::Type::kVideo) { + RTC_DCHECK(rtp_sequence_number_map_); + // Last packet of a frame, add it to sequence number info map. + const uint32_t timestamp = packet->Timestamp() - timestamp_offset_; + bool is_first_packet_of_frame = packet->is_first_packet_of_frame(); + bool is_last_packet_of_frame = packet->Marker(); + + rtp_sequence_number_map_->InsertPacket( + packet->SequenceNumber(), + RtpSequenceNumberMap::Info(timestamp, is_first_packet_of_frame, + is_last_packet_of_frame)); + } + } + + // Bug webrtc:7859. While FEC is invoked from rtp_sender_video, and not after + // the pacer, these modifications of the header below are happening after the + // FEC protection packets are calculated. This will corrupt recovered packets + // at the same place. It's not an issue for extensions, which are present in + // all the packets (their content just may be incorrect on recovered packets). + // In case of VideoTimingExtension, since it's present not in every packet, + // data after rtp header may be corrupted if these packets are protected by + // the FEC. + int64_t diff_ms = now_ms - packet->capture_time_ms(); + if (packet->HasExtension<TransmissionOffset>()) { + packet->SetExtension<TransmissionOffset>(kTimestampTicksPerMs * diff_ms); + } + if (packet->HasExtension<AbsoluteSendTime>()) { + packet->SetExtension<AbsoluteSendTime>( + AbsoluteSendTime::MsTo24Bits(now_ms)); + } + + if (packet->HasExtension<VideoTimingExtension>()) { + if (populate_network2_timestamp_) { + packet->set_network2_time_ms(now_ms); + } else { + packet->set_pacer_exit_time_ms(now_ms); + } + } + + const bool is_media = packet->packet_type() == RtpPacketMediaType::kAudio || + packet->packet_type() == RtpPacketMediaType::kVideo; + + // Downstream code actually uses this flag to distinguish between media and + // everything else. + options.is_retransmit = !is_media; + if (auto packet_id = packet->GetExtension<TransportSequenceNumber>()) { + options.packet_id = *packet_id; + options.included_in_feedback = true; + options.included_in_allocation = true; + AddPacketToTransportFeedback(*packet_id, *packet, pacing_info); + } + + options.application_data.assign(packet->application_data().begin(), + packet->application_data().end()); + + if (packet->packet_type() != RtpPacketMediaType::kPadding && + packet->packet_type() != RtpPacketMediaType::kRetransmission) { + UpdateDelayStatistics(packet->capture_time_ms(), now_ms, packet_ssrc); + UpdateOnSendPacket(options.packet_id, packet->capture_time_ms(), + packet_ssrc); + } + + const bool send_success = SendPacketToNetwork(*packet, options, pacing_info); + + // Put packet in retransmission history or update pending status even if + // actual sending fails. + if (is_media && packet->allow_retransmission()) { + packet_history_->PutRtpPacket(std::make_unique<RtpPacketToSend>(*packet), + now_ms); + } else if (packet->retransmitted_sequence_number()) { + packet_history_->MarkPacketAsSent(*packet->retransmitted_sequence_number()); + } + + if (send_success) { + rtc::CritScope lock(&lock_); + UpdateRtpStats(*packet); + media_has_been_sent_ = true; + } +} + +void DEPRECATED_RtpSenderEgress::ProcessBitrateAndNotifyObservers() { + if (!bitrate_callback_) + return; + + rtc::CritScope lock(&lock_); + RtpSendRates send_rates = GetSendRatesLocked(); + bitrate_callback_->Notify( + send_rates.Sum().bps(), + send_rates[RtpPacketMediaType::kRetransmission].bps(), ssrc_); +} + +RtpSendRates DEPRECATED_RtpSenderEgress::GetSendRates() const { + rtc::CritScope lock(&lock_); + return GetSendRatesLocked(); +} + +RtpSendRates DEPRECATED_RtpSenderEgress::GetSendRatesLocked() const { + const int64_t now_ms = clock_->TimeInMilliseconds(); + RtpSendRates current_rates; + for (size_t i = 0; i < kNumMediaTypes; ++i) { + RtpPacketMediaType type = static_cast<RtpPacketMediaType>(i); + current_rates[type] = + DataRate::BitsPerSec(send_rates_[i].Rate(now_ms).value_or(0)); + } + return current_rates; +} + +void DEPRECATED_RtpSenderEgress::GetDataCounters( + StreamDataCounters* rtp_stats, + StreamDataCounters* rtx_stats) const { + rtc::CritScope lock(&lock_); + *rtp_stats = rtp_stats_; + *rtx_stats = rtx_rtp_stats_; +} + +void DEPRECATED_RtpSenderEgress::ForceIncludeSendPacketsInAllocation( + bool part_of_allocation) { + rtc::CritScope lock(&lock_); + force_part_of_allocation_ = part_of_allocation; +} + +bool DEPRECATED_RtpSenderEgress::MediaHasBeenSent() const { + rtc::CritScope lock(&lock_); + return media_has_been_sent_; +} + +void DEPRECATED_RtpSenderEgress::SetMediaHasBeenSent(bool media_sent) { + rtc::CritScope lock(&lock_); + media_has_been_sent_ = media_sent; +} + +void DEPRECATED_RtpSenderEgress::SetTimestampOffset(uint32_t timestamp) { + rtc::CritScope lock(&lock_); + timestamp_offset_ = timestamp; +} + +std::vector<RtpSequenceNumberMap::Info> +DEPRECATED_RtpSenderEgress::GetSentRtpPacketInfos( + rtc::ArrayView<const uint16_t> sequence_numbers) const { + RTC_DCHECK(!sequence_numbers.empty()); + if (!need_rtp_packet_infos_) { + return std::vector<RtpSequenceNumberMap::Info>(); + } + + std::vector<RtpSequenceNumberMap::Info> results; + results.reserve(sequence_numbers.size()); + + rtc::CritScope cs(&lock_); + for (uint16_t sequence_number : sequence_numbers) { + const auto& info = rtp_sequence_number_map_->Get(sequence_number); + if (!info) { + // The empty vector will be returned. We can delay the clearing + // of the vector until after we exit the critical section. + return std::vector<RtpSequenceNumberMap::Info>(); + } + results.push_back(*info); + } + + return results; +} + +bool DEPRECATED_RtpSenderEgress::HasCorrectSsrc( + const RtpPacketToSend& packet) const { + switch (*packet.packet_type()) { + case RtpPacketMediaType::kAudio: + case RtpPacketMediaType::kVideo: + return packet.Ssrc() == ssrc_; + case RtpPacketMediaType::kRetransmission: + case RtpPacketMediaType::kPadding: + // Both padding and retransmission must be on either the media or the + // RTX stream. + return packet.Ssrc() == rtx_ssrc_ || packet.Ssrc() == ssrc_; + case RtpPacketMediaType::kForwardErrorCorrection: + // FlexFEC is on separate SSRC, ULPFEC uses media SSRC. + return packet.Ssrc() == ssrc_ || packet.Ssrc() == flexfec_ssrc_; + } + return false; +} + +void DEPRECATED_RtpSenderEgress::AddPacketToTransportFeedback( + uint16_t packet_id, + const RtpPacketToSend& packet, + const PacedPacketInfo& pacing_info) { + if (transport_feedback_observer_) { + size_t packet_size = packet.payload_size() + packet.padding_size(); + if (send_side_bwe_with_overhead_) { + packet_size = packet.size(); + } + + RtpPacketSendInfo packet_info; + packet_info.ssrc = ssrc_; + packet_info.transport_sequence_number = packet_id; + packet_info.rtp_sequence_number = packet.SequenceNumber(); + packet_info.length = packet_size; + packet_info.pacing_info = pacing_info; + packet_info.packet_type = packet.packet_type(); + transport_feedback_observer_->OnAddPacket(packet_info); + } +} + +void DEPRECATED_RtpSenderEgress::UpdateDelayStatistics(int64_t capture_time_ms, + int64_t now_ms, + uint32_t ssrc) { + if (!send_side_delay_observer_ || capture_time_ms <= 0) + return; + + int avg_delay_ms = 0; + int max_delay_ms = 0; + uint64_t total_packet_send_delay_ms = 0; + { + rtc::CritScope cs(&lock_); + // Compute the max and average of the recent capture-to-send delays. + // The time complexity of the current approach depends on the distribution + // of the delay values. This could be done more efficiently. + + // Remove elements older than kSendSideDelayWindowMs. + auto lower_bound = + send_delays_.lower_bound(now_ms - kSendSideDelayWindowMs); + for (auto it = send_delays_.begin(); it != lower_bound; ++it) { + if (max_delay_it_ == it) { + max_delay_it_ = send_delays_.end(); + } + sum_delays_ms_ -= it->second; + } + send_delays_.erase(send_delays_.begin(), lower_bound); + if (max_delay_it_ == send_delays_.end()) { + // Removed the previous max. Need to recompute. + RecomputeMaxSendDelay(); + } + + // Add the new element. + RTC_DCHECK_GE(now_ms, 0); + RTC_DCHECK_LE(now_ms, std::numeric_limits<int64_t>::max() / 2); + RTC_DCHECK_GE(capture_time_ms, 0); + RTC_DCHECK_LE(capture_time_ms, std::numeric_limits<int64_t>::max() / 2); + int64_t diff_ms = now_ms - capture_time_ms; + RTC_DCHECK_GE(diff_ms, static_cast<int64_t>(0)); + RTC_DCHECK_LE(diff_ms, std::numeric_limits<int>::max()); + int new_send_delay = rtc::dchecked_cast<int>(now_ms - capture_time_ms); + SendDelayMap::iterator it; + bool inserted; + std::tie(it, inserted) = + send_delays_.insert(std::make_pair(now_ms, new_send_delay)); + if (!inserted) { + // TODO(terelius): If we have multiple delay measurements during the same + // millisecond then we keep the most recent one. It is not clear that this + // is the right decision, but it preserves an earlier behavior. + int previous_send_delay = it->second; + sum_delays_ms_ -= previous_send_delay; + it->second = new_send_delay; + if (max_delay_it_ == it && new_send_delay < previous_send_delay) { + RecomputeMaxSendDelay(); + } + } + if (max_delay_it_ == send_delays_.end() || + it->second >= max_delay_it_->second) { + max_delay_it_ = it; + } + sum_delays_ms_ += new_send_delay; + total_packet_send_delay_ms_ += new_send_delay; + total_packet_send_delay_ms = total_packet_send_delay_ms_; + + size_t num_delays = send_delays_.size(); + RTC_DCHECK(max_delay_it_ != send_delays_.end()); + max_delay_ms = rtc::dchecked_cast<int>(max_delay_it_->second); + int64_t avg_ms = (sum_delays_ms_ + num_delays / 2) / num_delays; + RTC_DCHECK_GE(avg_ms, static_cast<int64_t>(0)); + RTC_DCHECK_LE(avg_ms, + static_cast<int64_t>(std::numeric_limits<int>::max())); + avg_delay_ms = + rtc::dchecked_cast<int>((sum_delays_ms_ + num_delays / 2) / num_delays); + } + send_side_delay_observer_->SendSideDelayUpdated( + avg_delay_ms, max_delay_ms, total_packet_send_delay_ms, ssrc); +} + +void DEPRECATED_RtpSenderEgress::RecomputeMaxSendDelay() { + max_delay_it_ = send_delays_.begin(); + for (auto it = send_delays_.begin(); it != send_delays_.end(); ++it) { + if (it->second >= max_delay_it_->second) { + max_delay_it_ = it; + } + } +} + +void DEPRECATED_RtpSenderEgress::UpdateOnSendPacket(int packet_id, + int64_t capture_time_ms, + uint32_t ssrc) { + if (!send_packet_observer_ || capture_time_ms <= 0 || packet_id == -1) { + return; + } + + send_packet_observer_->OnSendPacket(packet_id, capture_time_ms, ssrc); +} + +bool DEPRECATED_RtpSenderEgress::SendPacketToNetwork( + const RtpPacketToSend& packet, + const PacketOptions& options, + const PacedPacketInfo& pacing_info) { + int bytes_sent = -1; + if (transport_) { + bytes_sent = transport_->SendRtp(packet.data(), packet.size(), options) + ? static_cast<int>(packet.size()) + : -1; + if (event_log_ && bytes_sent > 0) { + event_log_->Log(std::make_unique<RtcEventRtpPacketOutgoing>( + packet, pacing_info.probe_cluster_id)); + } + } + + if (bytes_sent <= 0) { + RTC_LOG(LS_WARNING) << "Transport failed to send packet."; + return false; + } + return true; +} + +void DEPRECATED_RtpSenderEgress::UpdateRtpStats(const RtpPacketToSend& packet) { + int64_t now_ms = clock_->TimeInMilliseconds(); + + StreamDataCounters* counters = + packet.Ssrc() == rtx_ssrc_ ? &rtx_rtp_stats_ : &rtp_stats_; + + if (counters->first_packet_time_ms == -1) { + counters->first_packet_time_ms = now_ms; + } + + if (packet.packet_type() == RtpPacketMediaType::kForwardErrorCorrection) { + counters->fec.AddPacket(packet); + } + + if (packet.packet_type() == RtpPacketMediaType::kRetransmission) { + counters->retransmitted.AddPacket(packet); + } + counters->transmitted.AddPacket(packet); + + RTC_DCHECK(packet.packet_type().has_value()); + send_rates_[static_cast<size_t>(*packet.packet_type())].Update(packet.size(), + now_ms); + + if (rtp_stats_callback_) { + rtp_stats_callback_->DataCountersUpdated(*counters, packet.Ssrc()); + } +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.h new file mode 100644 index 00000000000..9f1d7d6c837 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_DEPRECATED_DEPRECATED_RTP_SENDER_EGRESS_H_ +#define MODULES_RTP_RTCP_SOURCE_DEPRECATED_DEPRECATED_RTP_SENDER_EGRESS_H_ + +#include <map> +#include <memory> +#include <vector> + +#include "absl/types/optional.h" +#include "api/call/transport.h" +#include "api/rtc_event_log/rtc_event_log.h" +#include "api/units/data_rate.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_packet_history.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/rate_statistics.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class DEPRECATED_RtpSenderEgress { + public: + // Helper class that redirects packets directly to the send part of this class + // without passing through an actual paced sender. + class NonPacedPacketSender : public RtpPacketSender { + public: + explicit NonPacedPacketSender(DEPRECATED_RtpSenderEgress* sender); + virtual ~NonPacedPacketSender(); + + void EnqueuePackets( + std::vector<std::unique_ptr<RtpPacketToSend>> packets) override; + + private: + uint16_t transport_sequence_number_; + DEPRECATED_RtpSenderEgress* const sender_; + }; + + DEPRECATED_RtpSenderEgress(const RtpRtcpInterface::Configuration& config, + RtpPacketHistory* packet_history); + ~DEPRECATED_RtpSenderEgress() = default; + + void SendPacket(RtpPacketToSend* packet, const PacedPacketInfo& pacing_info) + RTC_LOCKS_EXCLUDED(lock_); + uint32_t Ssrc() const { return ssrc_; } + absl::optional<uint32_t> RtxSsrc() const { return rtx_ssrc_; } + absl::optional<uint32_t> FlexFecSsrc() const { return flexfec_ssrc_; } + + void ProcessBitrateAndNotifyObservers() RTC_LOCKS_EXCLUDED(lock_); + RtpSendRates GetSendRates() const RTC_LOCKS_EXCLUDED(lock_); + void GetDataCounters(StreamDataCounters* rtp_stats, + StreamDataCounters* rtx_stats) const + RTC_LOCKS_EXCLUDED(lock_); + + void ForceIncludeSendPacketsInAllocation(bool part_of_allocation) + RTC_LOCKS_EXCLUDED(lock_); + bool MediaHasBeenSent() const RTC_LOCKS_EXCLUDED(lock_); + void SetMediaHasBeenSent(bool media_sent) RTC_LOCKS_EXCLUDED(lock_); + void SetTimestampOffset(uint32_t timestamp) RTC_LOCKS_EXCLUDED(lock_); + + // For each sequence number in |sequence_number|, recall the last RTP packet + // which bore it - its timestamp and whether it was the first and/or last + // packet in that frame. If all of the given sequence numbers could be + // recalled, return a vector with all of them (in corresponding order). + // If any could not be recalled, return an empty vector. + std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( + rtc::ArrayView<const uint16_t> sequence_numbers) const + RTC_LOCKS_EXCLUDED(lock_); + + private: + // Maps capture time in milliseconds to send-side delay in milliseconds. + // Send-side delay is the difference between transmission time and capture + // time. + typedef std::map<int64_t, int> SendDelayMap; + + RtpSendRates GetSendRatesLocked() const RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + bool HasCorrectSsrc(const RtpPacketToSend& packet) const; + void AddPacketToTransportFeedback(uint16_t packet_id, + const RtpPacketToSend& packet, + const PacedPacketInfo& pacing_info); + void UpdateDelayStatistics(int64_t capture_time_ms, + int64_t now_ms, + uint32_t ssrc); + void RecomputeMaxSendDelay() RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + void UpdateOnSendPacket(int packet_id, + int64_t capture_time_ms, + uint32_t ssrc); + // Sends packet on to |transport_|, leaving the RTP module. + bool SendPacketToNetwork(const RtpPacketToSend& packet, + const PacketOptions& options, + const PacedPacketInfo& pacing_info); + void UpdateRtpStats(const RtpPacketToSend& packet) + RTC_EXCLUSIVE_LOCKS_REQUIRED(lock_); + + const uint32_t ssrc_; + const absl::optional<uint32_t> rtx_ssrc_; + const absl::optional<uint32_t> flexfec_ssrc_; + const bool populate_network2_timestamp_; + const bool send_side_bwe_with_overhead_; + Clock* const clock_; + RtpPacketHistory* const packet_history_; + Transport* const transport_; + RtcEventLog* const event_log_; + const bool is_audio_; + const bool need_rtp_packet_infos_; + + TransportFeedbackObserver* const transport_feedback_observer_; + SendSideDelayObserver* const send_side_delay_observer_; + SendPacketObserver* const send_packet_observer_; + StreamDataCountersCallback* const rtp_stats_callback_; + BitrateStatisticsObserver* const bitrate_callback_; + + rtc::CriticalSection lock_; + bool media_has_been_sent_ RTC_GUARDED_BY(lock_); + bool force_part_of_allocation_ RTC_GUARDED_BY(lock_); + uint32_t timestamp_offset_ RTC_GUARDED_BY(lock_); + + SendDelayMap send_delays_ RTC_GUARDED_BY(lock_); + SendDelayMap::const_iterator max_delay_it_ RTC_GUARDED_BY(lock_); + // The sum of delays over a kSendSideDelayWindowMs sliding window. + int64_t sum_delays_ms_ RTC_GUARDED_BY(lock_); + uint64_t total_packet_send_delay_ms_ RTC_GUARDED_BY(lock_); + StreamDataCounters rtp_stats_ RTC_GUARDED_BY(lock_); + StreamDataCounters rtx_rtp_stats_ RTC_GUARDED_BY(lock_); + // One element per value in RtpPacketMediaType, with index matching value. + std::vector<RateStatistics> send_rates_ RTC_GUARDED_BY(lock_); + + // Maps sent packets' sequence numbers to a tuple consisting of: + // 1. The timestamp, without the randomizing offset mandated by the RFC. + // 2. Whether the packet was the first in its frame. + // 3. Whether the packet was the last in its frame. + const std::unique_ptr<RtpSequenceNumberMap> rtp_sequence_number_map_ + RTC_GUARDED_BY(lock_); +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_DEPRECATED_DEPRECATED_RTP_SENDER_EGRESS_H_ diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/nack_rtx_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/nack_rtx_unittest.cc index 55e1e44ebe4..c30eb32a44a 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/nack_rtx_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/nack_rtx_unittest.cc @@ -19,9 +19,9 @@ #include "call/rtp_stream_receiver_controller.h" #include "call/rtx_receive_stream.h" #include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/rtp_sender_video.h" #include "rtc_base/rate_limiter.h" #include "test/gtest.h" @@ -63,7 +63,9 @@ class RtxLoopBackTransport : public webrtc::Transport { count_rtx_ssrc_(0), module_(NULL) {} - void SetSendModule(RtpRtcp* rtpRtcpModule) { module_ = rtpRtcpModule; } + void SetSendModule(RtpRtcpInterface* rtpRtcpModule) { + module_ = rtpRtcpModule; + } void DropEveryNthPacket(int n) { packet_loss_ = n; } @@ -109,7 +111,7 @@ class RtxLoopBackTransport : public webrtc::Transport { int consecutive_drop_end_; uint32_t rtx_ssrc_; int count_rtx_ssrc_; - RtpRtcp* module_; + RtpRtcpInterface* module_; RtpStreamReceiverController stream_receiver_controller_; std::set<uint16_t> expected_sequence_numbers_; }; @@ -125,7 +127,7 @@ class RtpRtcpRtxNackTest : public ::testing::Test { ~RtpRtcpRtxNackTest() override {} void SetUp() override { - RtpRtcp::Configuration configuration; + RtpRtcpInterface::Configuration configuration; configuration.audio = false; configuration.clock = &fake_clock; receive_statistics_ = ReceiveStatistics::Create(&fake_clock); @@ -134,7 +136,7 @@ class RtpRtcpRtxNackTest : public ::testing::Test { configuration.retransmission_rate_limiter = &retransmission_rate_limiter_; configuration.local_media_ssrc = kTestSsrc; configuration.rtx_send_ssrc = kTestRtxSsrc; - rtp_rtcp_module_ = RtpRtcp::Create(configuration); + rtp_rtcp_module_ = ModuleRtpRtcpImpl2::Create(configuration); FieldTrialBasedConfig field_trials; RTPSenderVideo::Config video_config; video_config.clock = &fake_clock; @@ -224,7 +226,7 @@ class RtpRtcpRtxNackTest : public ::testing::Test { } std::unique_ptr<ReceiveStatistics> receive_statistics_; - std::unique_ptr<RtpRtcp> rtp_rtcp_module_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_module_; std::unique_ptr<RTPSenderVideo> rtp_sender_video_; RtxLoopBackTransport transport_; const std::map<int, int> rtx_associated_payload_types_ = { diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.cc index bfe26676849..da51a501f26 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.cc @@ -66,7 +66,8 @@ const size_t kMaxNumberOfStoredRrtrs = 300; constexpr int32_t kDefaultVideoReportInterval = 1000; constexpr int32_t kDefaultAudioReportInterval = 5000; -std::set<uint32_t> GetRegisteredSsrcs(const RtpRtcp::Configuration& config) { +std::set<uint32_t> GetRegisteredSsrcs( + const RtpRtcpInterface::Configuration& config) { std::set<uint32_t> ssrcs; ssrcs.insert(config.local_media_ssrc); if (config.rtx_send_ssrc) { @@ -136,7 +137,7 @@ struct RTCPReceiver::LastFirStatus { uint8_t sequence_number; }; -RTCPReceiver::RTCPReceiver(const RtpRtcp::Configuration& config, +RTCPReceiver::RTCPReceiver(const RtpRtcpInterface::Configuration& config, ModuleRtpRtcp* owner) : clock_(config.clock), receiver_only_(config.receiver_only), diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.h index ef41476903a..f7fb6075878 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver.h @@ -20,10 +20,10 @@ #include "api/array_view.h" #include "modules/rtp_rtcp/include/report_block_data.h" #include "modules/rtp_rtcp/include/rtcp_statistics.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtcp_nack_stats.h" #include "modules/rtp_rtcp/source/rtcp_packet/dlrr.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "rtc_base/critical_section.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/ntp_time.h" @@ -53,7 +53,8 @@ class RTCPReceiver final { virtual ~ModuleRtpRtcp() = default; }; - RTCPReceiver(const RtpRtcp::Configuration& config, ModuleRtpRtcp* owner); + RTCPReceiver(const RtpRtcpInterface::Configuration& config, + ModuleRtpRtcp* owner); ~RTCPReceiver(); void IncomingPacket(const uint8_t* packet, size_t packet_size) { diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc index f95219674b1..a384d71913b 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_receiver_unittest.cc @@ -161,8 +161,8 @@ struct ReceiverMocks { StrictMock<MockModuleRtpRtcp> rtp_rtcp_impl; }; -RtpRtcp::Configuration DefaultConfiguration(ReceiverMocks* mocks) { - RtpRtcp::Configuration config; +RtpRtcpInterface::Configuration DefaultConfiguration(ReceiverMocks* mocks) { + RtpRtcpInterface::Configuration config; config.clock = &mocks->clock; config.receiver_only = false; config.rtcp_packet_type_counter_observer = @@ -230,7 +230,7 @@ TEST(RtcpReceiverTest, InjectSrPacketFromUnknownSender) { rtcp::SenderReport sr; sr.SetSenderSsrc(kUnknownSenderSsrc); - // The parser will handle report blocks in Sender Report from other than his + // The parser will handle report blocks in Sender Report from other than their // expected peer. EXPECT_CALL(mocks.rtp_rtcp_impl, OnReceivedRtcpReportBlocks); EXPECT_CALL(mocks.bandwidth_observer, @@ -636,7 +636,7 @@ TEST(RtcpReceiverTest, InjectApp) { TEST(RtcpReceiverTest, InjectSdesWithOneChunk) { ReceiverMocks mocks; MockCnameCallbackImpl callback; - RtpRtcp::Configuration config = DefaultConfiguration(&mocks); + RtpRtcpInterface::Configuration config = DefaultConfiguration(&mocks); config.rtcp_cname_callback = &callback; RTCPReceiver receiver(config, &mocks.rtp_rtcp_impl); receiver.SetRemoteSSRC(kSenderSsrc); @@ -1310,7 +1310,7 @@ TEST(RtcpReceiverTest, TmmbrThreeConstraintsTimeOut) { TEST(RtcpReceiverTest, Callbacks) { ReceiverMocks mocks; MockRtcpCallbackImpl callback; - RtpRtcp::Configuration config = DefaultConfiguration(&mocks); + RtpRtcpInterface::Configuration config = DefaultConfiguration(&mocks); config.rtcp_statistics_callback = &callback; RTCPReceiver receiver(config, &mocks.rtp_rtcp_impl); receiver.SetRemoteSSRC(kSenderSsrc); @@ -1348,7 +1348,7 @@ TEST(RtcpReceiverTest, VerifyBlockAndTimestampObtainedFromReportBlockDataObserver) { ReceiverMocks mocks; MockReportBlockDataObserverImpl observer; - RtpRtcp::Configuration config = DefaultConfiguration(&mocks); + RtpRtcpInterface::Configuration config = DefaultConfiguration(&mocks); config.report_block_data_observer = &observer; RTCPReceiver receiver(config, &mocks.rtp_rtcp_impl); receiver.SetRemoteSSRC(kSenderSsrc); @@ -1397,7 +1397,7 @@ TEST(RtcpReceiverTest, TEST(RtcpReceiverTest, VerifyRttObtainedFromReportBlockDataObserver) { ReceiverMocks mocks; MockReportBlockDataObserverImpl observer; - RtpRtcp::Configuration config = DefaultConfiguration(&mocks); + RtpRtcpInterface::Configuration config = DefaultConfiguration(&mocks); config.report_block_data_observer = &observer; RTCPReceiver receiver(config, &mocks.rtp_rtcp_impl); receiver.SetRemoteSSRC(kSenderSsrc); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.cc index f06d429fb92..f3e04b17f3c 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.cc @@ -33,7 +33,7 @@ #include "modules/rtp_rtcp/source/rtcp_packet/tmmbn.h" #include "modules/rtp_rtcp/source/rtcp_packet/tmmbr.h" #include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h" -#include "modules/rtp_rtcp/source/rtp_rtcp_impl.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/time_util.h" #include "modules/rtp_rtcp/source/tmmbr_help.h" #include "rtc_base/checks.h" @@ -123,7 +123,7 @@ RTCPSender::FeedbackState::FeedbackState() last_rr_ntp_secs(0), last_rr_ntp_frac(0), remote_sr(0), - module(nullptr) {} + receiver(nullptr) {} RTCPSender::FeedbackState::FeedbackState(const FeedbackState&) = default; @@ -148,7 +148,7 @@ class RTCPSender::RtcpContext { const int64_t now_us_; }; -RTCPSender::RTCPSender(const RtpRtcp::Configuration& config) +RTCPSender::RTCPSender(const RtpRtcpInterface::Configuration& config) : audio_(config.audio), ssrc_(config.local_media_ssrc), clock_(config.clock), @@ -176,11 +176,6 @@ RTCPSender::RTCPSender(const RtpRtcp::Configuration& config) packet_oh_send_(0), max_packet_size_(IP_PACKET_SIZE - 28), // IPv4 + UDP by default. - app_sub_type_(0), - app_name_(0), - app_data_(nullptr), - app_length_(0), - xr_send_receiver_reference_time_enabled_(false), packet_type_counter_observer_(config.rtcp_packet_type_counter_observer), send_video_bitrate_allocation_(false), @@ -194,7 +189,6 @@ RTCPSender::RTCPSender(const RtpRtcp::Configuration& config) builders_[kRtcpFir] = &RTCPSender::BuildFIR; builders_[kRtcpRemb] = &RTCPSender::BuildREMB; builders_[kRtcpBye] = &RTCPSender::BuildBYE; - builders_[kRtcpApp] = &RTCPSender::BuildAPP; builders_[kRtcpLossNotification] = &RTCPSender::BuildLossNotification; builders_[kRtcpTmmbr] = &RTCPSender::BuildTMMBR; builders_[kRtcpTmmbn] = &RTCPSender::BuildTMMBN; @@ -262,8 +256,8 @@ int32_t RTCPSender::SendLossNotification(const FeedbackState& feedback_state, return 0; } - return SendCompoundRTCP(feedback_state, - {RTCPPacketType::kRtcpLossNotification}); + return SendCompoundRTCPLocked( + feedback_state, {RTCPPacketType::kRtcpLossNotification}, 0, nullptr); } void RTCPSender::SetRemb(int64_t bitrate_bps, std::vector<uint32_t> ssrcs) { @@ -544,7 +538,7 @@ void RTCPSender::SetTargetBitrate(unsigned int target_bitrate) { std::unique_ptr<rtcp::RtcpPacket> RTCPSender::BuildTMMBR( const RtcpContext& ctx) { - if (ctx.feedback_state_.module == nullptr) + if (ctx.feedback_state_.receiver == nullptr) return nullptr; // Before sending the TMMBR check the received TMMBN, only an owner is // allowed to raise the bitrate: @@ -558,7 +552,7 @@ std::unique_ptr<rtcp::RtcpPacket> RTCPSender::BuildTMMBR( // will accuire criticalSectionRTCPReceiver_ is a potental deadlock but // since RTCPreceiver is not doing the reverse we should be fine std::vector<rtcp::TmmbItem> candidates = - ctx.feedback_state_.module->BoundingSet(&tmmbr_owner); + ctx.feedback_state_.receiver->BoundingSet(&tmmbr_owner); if (!candidates.empty()) { for (const auto& candidate : candidates) { @@ -614,9 +608,6 @@ std::unique_ptr<rtcp::RtcpPacket> RTCPSender::BuildTMMBN( std::unique_ptr<rtcp::RtcpPacket> RTCPSender::BuildAPP(const RtcpContext& ctx) { rtcp::App* app = new rtcp::App(); app->SetSenderSsrc(ssrc_); - app->SetSubType(app_sub_type_); - app->SetName(app_name_); - app->SetData(app_data_.get(), app_length_); return std::unique_ptr<rtcp::RtcpPacket>(app); } @@ -712,52 +703,86 @@ int32_t RTCPSender::SendCompoundRTCP( { rtc::CritScope lock(&critical_section_rtcp_sender_); - if (method_ == RtcpMode::kOff) { - RTC_LOG(LS_WARNING) << "Can't send rtcp if it is disabled."; - return -1; + auto result = ComputeCompoundRTCPPacket(feedback_state, packet_types, + nack_size, nack_list, &container); + if (result) { + return *result; } - // Add all flags as volatile. Non volatile entries will not be overwritten. - // All new volatile flags added will be consumed by the end of this call. - SetFlags(packet_types, true); - - // Prevent sending streams to send SR before any media has been sent. - const bool can_calculate_rtp_timestamp = (last_frame_capture_time_ms_ >= 0); - if (!can_calculate_rtp_timestamp) { - bool consumed_sr_flag = ConsumeFlag(kRtcpSr); - bool consumed_report_flag = sending_ && ConsumeFlag(kRtcpReport); - bool sender_report = consumed_report_flag || consumed_sr_flag; - if (sender_report && AllVolatileFlagsConsumed()) { - // This call was for Sender Report and nothing else. - return 0; - } - if (sending_ && method_ == RtcpMode::kCompound) { - // Not allowed to send any RTCP packet without sender report. - return -1; - } + max_packet_size = max_packet_size_; + } + + size_t bytes_sent = container.SendPackets(max_packet_size); + return bytes_sent == 0 ? -1 : 0; +} + +int32_t RTCPSender::SendCompoundRTCPLocked( + const FeedbackState& feedback_state, + const std::set<RTCPPacketType>& packet_types, + int32_t nack_size, + const uint16_t* nack_list) { + PacketContainer container(transport_, event_log_); + auto result = ComputeCompoundRTCPPacket(feedback_state, packet_types, + nack_size, nack_list, &container); + if (result) { + return *result; + } + size_t bytes_sent = container.SendPackets(max_packet_size_); + return bytes_sent == 0 ? -1 : 0; +} + +absl::optional<int32_t> RTCPSender::ComputeCompoundRTCPPacket( + const FeedbackState& feedback_state, + const std::set<RTCPPacketType>& packet_types, + int32_t nack_size, + const uint16_t* nack_list, + rtcp::CompoundPacket* out_packet) { + if (method_ == RtcpMode::kOff) { + RTC_LOG(LS_WARNING) << "Can't send rtcp if it is disabled."; + return -1; + } + // Add all flags as volatile. Non volatile entries will not be overwritten. + // All new volatile flags added will be consumed by the end of this call. + SetFlags(packet_types, true); + + // Prevent sending streams to send SR before any media has been sent. + const bool can_calculate_rtp_timestamp = (last_frame_capture_time_ms_ >= 0); + if (!can_calculate_rtp_timestamp) { + bool consumed_sr_flag = ConsumeFlag(kRtcpSr); + bool consumed_report_flag = sending_ && ConsumeFlag(kRtcpReport); + bool sender_report = consumed_report_flag || consumed_sr_flag; + if (sender_report && AllVolatileFlagsConsumed()) { + // This call was for Sender Report and nothing else. + return 0; } + if (sending_ && method_ == RtcpMode::kCompound) { + // Not allowed to send any RTCP packet without sender report. + return -1; + } + } - if (packet_type_counter_.first_packet_time_ms == -1) - packet_type_counter_.first_packet_time_ms = clock_->TimeInMilliseconds(); + if (packet_type_counter_.first_packet_time_ms == -1) + packet_type_counter_.first_packet_time_ms = clock_->TimeInMilliseconds(); - // We need to send our NTP even if we haven't received any reports. - RtcpContext context(feedback_state, nack_size, nack_list, - clock_->TimeInMicroseconds()); + // We need to send our NTP even if we haven't received any reports. + RtcpContext context(feedback_state, nack_size, nack_list, + clock_->TimeInMicroseconds()); - PrepareReport(feedback_state); + PrepareReport(feedback_state); - std::unique_ptr<rtcp::RtcpPacket> packet_bye; + std::unique_ptr<rtcp::RtcpPacket> packet_bye; - auto it = report_flags_.begin(); - while (it != report_flags_.end()) { - auto builder_it = builders_.find(it->type); - RTC_DCHECK(builder_it != builders_.end()) - << "Could not find builder for packet type " << it->type; - if (it->is_volatile) { - report_flags_.erase(it++); - } else { - ++it; - } + auto it = report_flags_.begin(); + while (it != report_flags_.end()) { + auto builder_it = builders_.find(it->type); + if (it->is_volatile) { + report_flags_.erase(it++); + } else { + ++it; + } + if (builder_it == builders_.end()) { + RTC_NOTREACHED() << "Could not find builder for packet type " << it->type; + } else { BuilderFunc func = builder_it->second; std::unique_ptr<rtcp::RtcpPacket> packet = (this->*func)(context); if (packet == nullptr) @@ -767,26 +792,23 @@ int32_t RTCPSender::SendCompoundRTCP( if (builder_it->first == kRtcpBye) { packet_bye = std::move(packet); } else { - container.Append(packet.release()); + out_packet->Append(packet.release()); } } + } - // Append the BYE now at the end - if (packet_bye) { - container.Append(packet_bye.release()); - } - - if (packet_type_counter_observer_ != nullptr) { - packet_type_counter_observer_->RtcpPacketTypesCounterUpdated( - remote_ssrc_, packet_type_counter_); - } + // Append the BYE now at the end + if (packet_bye) { + out_packet->Append(packet_bye.release()); + } - RTC_DCHECK(AllVolatileFlagsConsumed()); - max_packet_size = max_packet_size_; + if (packet_type_counter_observer_ != nullptr) { + packet_type_counter_observer_->RtcpPacketTypesCounterUpdated( + remote_ssrc_, packet_type_counter_); } - size_t bytes_sent = container.SendPackets(max_packet_size); - return bytes_sent == 0 ? -1 : 0; + RTC_DCHECK(AllVolatileFlagsConsumed()); + return absl::nullopt; } void RTCPSender::PrepareReport(const FeedbackState& feedback_state) { @@ -877,25 +899,6 @@ void RTCPSender::SetCsrcs(const std::vector<uint32_t>& csrcs) { csrcs_ = csrcs; } -int32_t RTCPSender::SetApplicationSpecificData(uint8_t subType, - uint32_t name, - const uint8_t* data, - uint16_t length) { - if (length % 4 != 0) { - RTC_LOG(LS_ERROR) << "Failed to SetApplicationSpecificData."; - return -1; - } - rtc::CritScope lock(&critical_section_rtcp_sender_); - - SetFlag(kRtcpApp, true); - app_sub_type_ = subType; - app_name_ = name; - app_data_.reset(new uint8_t[length]); - app_length_ = length; - memcpy(app_data_.get(), data, length); - return 0; -} - void RTCPSender::SendRtcpXrReceiverReferenceTime(bool enable) { rtc::CritScope lock(&critical_section_rtcp_sender_); xr_send_receiver_reference_time_enabled_ = enable; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.h index 32c1e1dbc10..d9422ebe8e0 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender.h @@ -23,13 +23,14 @@ #include "modules/remote_bitrate_estimator/include/bwe_defines.h" #include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "modules/rtp_rtcp/include/receive_statistics.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtcp_nack_stats.h" #include "modules/rtp_rtcp/source/rtcp_packet.h" +#include "modules/rtp_rtcp/source/rtcp_packet/compound_packet.h" #include "modules/rtp_rtcp/source/rtcp_packet/dlrr.h" #include "modules/rtp_rtcp/source/rtcp_packet/report_block.h" #include "modules/rtp_rtcp/source/rtcp_packet/tmmb_item.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/critical_section.h" #include "rtc_base/random.h" @@ -37,10 +38,10 @@ namespace webrtc { -class ModuleRtpRtcpImpl; +class RTCPReceiver; class RtcEventLog; -class RTCPSender { +class RTCPSender final { public: struct FeedbackState { FeedbackState(); @@ -60,90 +61,124 @@ class RTCPSender { std::vector<rtcp::ReceiveTimeInfo> last_xr_rtis; // Used when generating TMMBR. - ModuleRtpRtcpImpl* module; + RTCPReceiver* receiver; }; - explicit RTCPSender(const RtpRtcp::Configuration& config); + explicit RTCPSender(const RtpRtcpInterface::Configuration& config); virtual ~RTCPSender(); - RtcpMode Status() const; - void SetRTCPStatus(RtcpMode method); + RtcpMode Status() const RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); + void SetRTCPStatus(RtcpMode method) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - bool Sending() const; + bool Sending() const RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); int32_t SetSendingStatus(const FeedbackState& feedback_state, - bool enabled); // combine the functions + bool enabled) + RTC_LOCKS_EXCLUDED( + critical_section_rtcp_sender_); // combine the functions - int32_t SetNackStatus(bool enable); + int32_t SetNackStatus(bool enable) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetTimestampOffset(uint32_t timestamp_offset); + void SetTimestampOffset(uint32_t timestamp_offset) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); // TODO(bugs.webrtc.org/6458): Remove default parameter value when all the // depending projects are updated to correctly set payload type. void SetLastRtpTime(uint32_t rtp_timestamp, int64_t capture_time_ms, - int8_t payload_type = -1); + int8_t payload_type = -1) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetRtpClockRate(int8_t payload_type, int rtp_clock_rate_hz); + void SetRtpClockRate(int8_t payload_type, int rtp_clock_rate_hz) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); uint32_t SSRC() const { return ssrc_; } - void SetRemoteSSRC(uint32_t ssrc); + void SetRemoteSSRC(uint32_t ssrc) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - int32_t SetCNAME(const char* cName); + int32_t SetCNAME(const char* cName) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - int32_t AddMixedCNAME(uint32_t SSRC, const char* c_name); + int32_t AddMixedCNAME(uint32_t SSRC, const char* c_name) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - int32_t RemoveMixedCNAME(uint32_t SSRC); + int32_t RemoveMixedCNAME(uint32_t SSRC) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - bool TimeToSendRTCPReport(bool sendKeyframeBeforeRTP = false) const; + bool TimeToSendRTCPReport(bool sendKeyframeBeforeRTP = false) const + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); int32_t SendRTCP(const FeedbackState& feedback_state, RTCPPacketType packetType, int32_t nackSize = 0, - const uint16_t* nackList = 0); + const uint16_t* nackList = 0) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); int32_t SendCompoundRTCP(const FeedbackState& feedback_state, const std::set<RTCPPacketType>& packetTypes, int32_t nackSize = 0, - const uint16_t* nackList = 0); + const uint16_t* nackList = nullptr) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); int32_t SendLossNotification(const FeedbackState& feedback_state, uint16_t last_decoded_seq_num, uint16_t last_received_seq_num, bool decodability_flag, - bool buffering_allowed); + bool buffering_allowed) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetRemb(int64_t bitrate_bps, std::vector<uint32_t> ssrcs); + void SetRemb(int64_t bitrate_bps, std::vector<uint32_t> ssrcs) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void UnsetRemb(); + void UnsetRemb() RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - bool TMMBR() const; + bool TMMBR() const RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetTMMBRStatus(bool enable); + void SetTMMBRStatus(bool enable) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetMaxRtpPacketSize(size_t max_packet_size); + void SetMaxRtpPacketSize(size_t max_packet_size) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetTmmbn(std::vector<rtcp::TmmbItem> bounding_set); + void SetTmmbn(std::vector<rtcp::TmmbItem> bounding_set) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - int32_t SetApplicationSpecificData(uint8_t subType, - uint32_t name, - const uint8_t* data, - uint16_t length); + void SendRtcpXrReceiverReferenceTime(bool enable) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SendRtcpXrReceiverReferenceTime(bool enable); + bool RtcpXrReceiverReferenceTime() const + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - bool RtcpXrReceiverReferenceTime() const; + void SetCsrcs(const std::vector<uint32_t>& csrcs) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); - void SetCsrcs(const std::vector<uint32_t>& csrcs); - - void SetTargetBitrate(unsigned int target_bitrate); - void SetVideoBitrateAllocation(const VideoBitrateAllocation& bitrate); + void SetTargetBitrate(unsigned int target_bitrate) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); + void SetVideoBitrateAllocation(const VideoBitrateAllocation& bitrate) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); void SendCombinedRtcpPacket( - std::vector<std::unique_ptr<rtcp::RtcpPacket>> rtcp_packets); + std::vector<std::unique_ptr<rtcp::RtcpPacket>> rtcp_packets) + RTC_LOCKS_EXCLUDED(critical_section_rtcp_sender_); private: class RtcpContext; + int32_t SendCompoundRTCPLocked(const FeedbackState& feedback_state, + const std::set<RTCPPacketType>& packet_types, + int32_t nack_size, + const uint16_t* nack_list) + RTC_EXCLUSIVE_LOCKS_REQUIRED(critical_section_rtcp_sender_); + + absl::optional<int32_t> ComputeCompoundRTCPPacket( + const FeedbackState& feedback_state, + const std::set<RTCPPacketType>& packet_types, + int32_t nack_size, + const uint16_t* nack_list, + rtcp::CompoundPacket* out_packet) + RTC_EXCLUSIVE_LOCKS_REQUIRED(critical_section_rtcp_sender_); + // Determine which RTCP messages should be sent and setup flags. void PrepareReport(const FeedbackState& feedback_state) RTC_EXCLUSIVE_LOCKS_REQUIRED(critical_section_rtcp_sender_); @@ -237,13 +272,6 @@ class RTCPSender { uint32_t packet_oh_send_ RTC_GUARDED_BY(critical_section_rtcp_sender_); size_t max_packet_size_ RTC_GUARDED_BY(critical_section_rtcp_sender_); - // APP - uint8_t app_sub_type_ RTC_GUARDED_BY(critical_section_rtcp_sender_); - uint32_t app_name_ RTC_GUARDED_BY(critical_section_rtcp_sender_); - std::unique_ptr<uint8_t[]> app_data_ - RTC_GUARDED_BY(critical_section_rtcp_sender_); - uint16_t app_length_ RTC_GUARDED_BY(critical_section_rtcp_sender_); - // True if sending of XR Receiver reference time report is enabled. bool xr_send_receiver_reference_time_enabled_ RTC_GUARDED_BY(critical_section_rtcp_sender_); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender_unittest.cc index 754ad893271..4b6d4a3da94 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_sender_unittest.cc @@ -18,7 +18,7 @@ #include "modules/rtp_rtcp/source/rtcp_packet/bye.h" #include "modules/rtp_rtcp/source/rtcp_packet/common_header.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" -#include "modules/rtp_rtcp/source/rtp_rtcp_impl.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/time_util.h" #include "rtc_base/rate_limiter.h" #include "test/gmock.h" @@ -76,8 +76,8 @@ class RtcpSenderTest : public ::testing::Test { : clock_(1335900000), receive_statistics_(ReceiveStatistics::Create(&clock_)), retransmission_rate_limiter_(&clock_, 1000) { - RtpRtcp::Configuration configuration = GetDefaultConfig(); - rtp_rtcp_impl_.reset(new ModuleRtpRtcpImpl(configuration)); + RtpRtcpInterface::Configuration configuration = GetDefaultConfig(); + rtp_rtcp_impl_.reset(new ModuleRtpRtcpImpl2(configuration)); rtcp_sender_.reset(new RTCPSender(configuration)); rtcp_sender_->SetRemoteSSRC(kRemoteSsrc); rtcp_sender_->SetTimestampOffset(kStartRtpTimestamp); @@ -85,8 +85,8 @@ class RtcpSenderTest : public ::testing::Test { /*payload_type=*/0); } - RtpRtcp::Configuration GetDefaultConfig() { - RtpRtcp::Configuration configuration; + RtpRtcpInterface::Configuration GetDefaultConfig() { + RtpRtcpInterface::Configuration configuration; configuration.audio = false; configuration.clock = &clock_; configuration.outgoing_transport = &test_transport_; @@ -115,7 +115,7 @@ class RtcpSenderTest : public ::testing::Test { SimulatedClock clock_; TestTransport test_transport_; std::unique_ptr<ReceiveStatistics> receive_statistics_; - std::unique_ptr<ModuleRtpRtcpImpl> rtp_rtcp_impl_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_rtcp_impl_; std::unique_ptr<RTCPSender> rtcp_sender_; RateLimiter retransmission_rate_limiter_; }; @@ -191,7 +191,7 @@ TEST_F(RtcpSenderTest, SendConsecutiveSrWithExactSlope) { } TEST_F(RtcpSenderTest, DoNotSendSrBeforeRtp) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &clock_; config.receive_statistics = receive_statistics_.get(); config.outgoing_transport = &test_transport_; @@ -213,7 +213,7 @@ TEST_F(RtcpSenderTest, DoNotSendSrBeforeRtp) { } TEST_F(RtcpSenderTest, DoNotSendCompundBeforeRtp) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &clock_; config.receive_statistics = receive_statistics_.get(); config.outgoing_transport = &test_transport_; @@ -315,47 +315,6 @@ TEST_F(RtcpSenderTest, StopSendingTriggersBye) { EXPECT_EQ(kSenderSsrc, parser()->bye()->sender_ssrc()); } -TEST_F(RtcpSenderTest, SendApp) { - const uint8_t kSubType = 30; - uint32_t name = 'n' << 24; - name += 'a' << 16; - name += 'm' << 8; - name += 'e'; - const uint8_t kData[] = {'t', 'e', 's', 't', 'd', 'a', 't', 'a'}; - EXPECT_EQ(0, rtcp_sender_->SetApplicationSpecificData(kSubType, name, kData, - sizeof(kData))); - rtcp_sender_->SetRTCPStatus(RtcpMode::kReducedSize); - EXPECT_EQ(0, rtcp_sender_->SendRTCP(feedback_state(), kRtcpApp)); - EXPECT_EQ(1, parser()->app()->num_packets()); - EXPECT_EQ(kSubType, parser()->app()->sub_type()); - EXPECT_EQ(name, parser()->app()->name()); - EXPECT_EQ(sizeof(kData), parser()->app()->data_size()); - EXPECT_EQ(0, memcmp(kData, parser()->app()->data(), sizeof(kData))); -} - -TEST_F(RtcpSenderTest, SendEmptyApp) { - const uint8_t kSubType = 30; - const uint32_t kName = 0x6E616D65; - - EXPECT_EQ( - 0, rtcp_sender_->SetApplicationSpecificData(kSubType, kName, nullptr, 0)); - - rtcp_sender_->SetRTCPStatus(RtcpMode::kReducedSize); - EXPECT_EQ(0, rtcp_sender_->SendRTCP(feedback_state(), kRtcpApp)); - EXPECT_EQ(1, parser()->app()->num_packets()); - EXPECT_EQ(kSubType, parser()->app()->sub_type()); - EXPECT_EQ(kName, parser()->app()->name()); - EXPECT_EQ(0U, parser()->app()->data_size()); -} - -TEST_F(RtcpSenderTest, SetInvalidApplicationSpecificData) { - const uint8_t kData[] = {'t', 'e', 's', 't', 'd', 'a', 't'}; - const uint16_t kInvalidDataLength = sizeof(kData) / sizeof(kData[0]); - EXPECT_EQ(-1, - rtcp_sender_->SetApplicationSpecificData( - 0, 0, kData, kInvalidDataLength)); // Should by multiple of 4. -} - TEST_F(RtcpSenderTest, SendFir) { rtcp_sender_->SetRTCPStatus(RtcpMode::kReducedSize); EXPECT_EQ(0, rtcp_sender_->SendRTCP(feedback_state(), kRtcpFir)); @@ -563,7 +522,7 @@ TEST_F(RtcpSenderTest, TestNoXrRrtrSentIfNotEnabled) { TEST_F(RtcpSenderTest, TestRegisterRtcpPacketTypeObserver) { RtcpPacketTypeCounterObserverImpl observer; - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &clock_; config.receive_statistics = receive_statistics_.get(); config.outgoing_transport = &test_transport_; @@ -691,7 +650,7 @@ TEST_F(RtcpSenderTest, ByeMustBeLast) { })); // Re-configure rtcp_sender_ with mock_transport_ - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &clock_; config.receive_statistics = receive_statistics_.get(); config.outgoing_transport = &mock_transport; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_config.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_config.h index 2cbd1045d27..8a8fd6aed8f 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_config.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_config.h @@ -28,8 +28,8 @@ class MediaReceiverRtcpObserver { public: virtual ~MediaReceiverRtcpObserver() = default; - // All message handlers have default empty implementation. This way user needs - // to implement only those she is interested in. + // All message handlers have default empty implementation. This way users only + // need to implement the ones they are interested in. virtual void OnSenderReport(uint32_t sender_ssrc, NtpTime ntp_time, uint32_t rtp_time) {} diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc index 727a9bca231..9c4c5adf79f 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_impl_unittest.cc @@ -55,15 +55,17 @@ using ::webrtc::test::RtcpPacketParser; class MockReceiveStatisticsProvider : public webrtc::ReceiveStatisticsProvider { public: - MOCK_METHOD1(RtcpReportBlocks, std::vector<ReportBlock>(size_t)); + MOCK_METHOD(std::vector<ReportBlock>, RtcpReportBlocks, (size_t), (override)); }; class MockMediaReceiverRtcpObserver : public webrtc::MediaReceiverRtcpObserver { public: - MOCK_METHOD3(OnSenderReport, void(uint32_t, NtpTime, uint32_t)); - MOCK_METHOD1(OnBye, void(uint32_t)); - MOCK_METHOD2(OnBitrateAllocation, - void(uint32_t, const VideoBitrateAllocation&)); + MOCK_METHOD(void, OnSenderReport, (uint32_t, NtpTime, uint32_t), (override)); + MOCK_METHOD(void, OnBye, (uint32_t), (override)); + MOCK_METHOD(void, + OnBitrateAllocation, + (uint32_t, const VideoBitrateAllocation&), + (override)); }; // Since some tests will need to wait for this period, make it small to avoid diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_unittest.cc index 5fb2aa55eb2..9c181c65266 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtcp_transceiver_unittest.cc @@ -42,7 +42,10 @@ using ::webrtc::test::RtcpPacketParser; class MockMediaReceiverRtcpObserver : public webrtc::MediaReceiverRtcpObserver { public: - MOCK_METHOD3(OnSenderReport, void(uint32_t, webrtc::NtpTime, uint32_t)); + MOCK_METHOD(void, + OnSenderReport, + (uint32_t, webrtc::NtpTime, uint32_t), + (override)); }; constexpr int kTimeoutMs = 1000; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc index 30dedb192f2..3b098185766 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.cc @@ -10,6 +10,7 @@ #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" +#include <bitset> #include <cstdint> #include "api/array_view.h" @@ -23,6 +24,7 @@ namespace webrtc { constexpr RTPExtensionType RtpDependencyDescriptorExtension::kId; constexpr char RtpDependencyDescriptorExtension::kUri[]; +constexpr std::bitset<32> RtpDependencyDescriptorExtension::kAllChainsAreActive; bool RtpDependencyDescriptorExtension::Parse( rtc::ArrayView<const uint8_t> data, @@ -34,16 +36,20 @@ bool RtpDependencyDescriptorExtension::Parse( size_t RtpDependencyDescriptorExtension::ValueSize( const FrameDependencyStructure& structure, + std::bitset<32> active_chains, const DependencyDescriptor& descriptor) { - RtpDependencyDescriptorWriter writer(/*data=*/{}, structure, descriptor); + RtpDependencyDescriptorWriter writer(/*data=*/{}, structure, active_chains, + descriptor); return DivideRoundUp(writer.ValueSizeBits(), 8); } bool RtpDependencyDescriptorExtension::Write( rtc::ArrayView<uint8_t> data, const FrameDependencyStructure& structure, + std::bitset<32> active_chains, const DependencyDescriptor& descriptor) { - RtpDependencyDescriptorWriter writer(data, structure, descriptor); + RtpDependencyDescriptorWriter writer(data, structure, active_chains, + descriptor); return writer.Write(); } diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h index b99230c56b3..de16eeab2a3 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h @@ -10,6 +10,7 @@ #ifndef MODULES_RTP_RTCP_SOURCE_RTP_DEPENDENCY_DESCRIPTOR_EXTENSION_H_ #define MODULES_RTP_RTCP_SOURCE_RTP_DEPENDENCY_DESCRIPTOR_EXTENSION_H_ +#include <bitset> #include <cstdint> #include "api/array_view.h" @@ -34,10 +35,24 @@ class RtpDependencyDescriptorExtension { DependencyDescriptor* descriptor); static size_t ValueSize(const FrameDependencyStructure& structure, + const DependencyDescriptor& descriptor) { + return ValueSize(structure, kAllChainsAreActive, descriptor); + } + static size_t ValueSize(const FrameDependencyStructure& structure, + std::bitset<32> active_chains, const DependencyDescriptor& descriptor); static bool Write(rtc::ArrayView<uint8_t> data, const FrameDependencyStructure& structure, + const DependencyDescriptor& descriptor) { + return Write(data, structure, kAllChainsAreActive, descriptor); + } + static bool Write(rtc::ArrayView<uint8_t> data, + const FrameDependencyStructure& structure, + std::bitset<32> active_chains, const DependencyDescriptor& descriptor); + + private: + static constexpr std::bitset<32> kAllChainsAreActive = ~uint32_t{0}; }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension_unittest.cc new file mode 100644 index 00000000000..11d809693c7 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_extension_unittest.cc @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" + +#include "api/array_view.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" + +#include "test/gmock.h" + +namespace webrtc { +namespace { + +using ::testing::Each; + +TEST(RtpDependencyDescriptorExtensionTest, Writer3BytesForPerfectTemplate) { + uint8_t buffer[3]; + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.templates = { + FrameDependencyTemplate().Dtis("SR").FrameDiffs({1}).ChainDiffs({2, 2})}; + DependencyDescriptor descriptor; + descriptor.frame_dependencies = structure.templates[0]; + + EXPECT_EQ(RtpDependencyDescriptorExtension::ValueSize(structure, descriptor), + 3u); + EXPECT_TRUE( + RtpDependencyDescriptorExtension::Write(buffer, structure, descriptor)); +} + +TEST(RtpDependencyDescriptorExtensionTest, WriteZeroInUnusedBits) { + uint8_t buffer[32]; + std::memset(buffer, 0xff, sizeof(buffer)); + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.templates = { + FrameDependencyTemplate().Dtis("SR").FrameDiffs({1}).ChainDiffs({1, 1})}; + DependencyDescriptor descriptor; + descriptor.frame_dependencies = structure.templates[0]; + descriptor.frame_dependencies.frame_diffs = {2}; + + // To test unused bytes are zeroed, need a buffer large enough. + size_t value_size = + RtpDependencyDescriptorExtension::ValueSize(structure, descriptor); + ASSERT_LT(value_size, sizeof(buffer)); + + ASSERT_TRUE( + RtpDependencyDescriptorExtension::Write(buffer, structure, descriptor)); + + const uint8_t* unused_bytes = buffer + value_size; + size_t num_unused_bytes = buffer + sizeof(buffer) - unused_bytes; + // Check remaining bytes are zeroed. + EXPECT_THAT(rtc::MakeArrayView(unused_bytes, num_unused_bytes), Each(0)); +} + +// In practice chain diff for inactive chain will grow uboundly because no +// frames are produced for it, that shouldn't block writing the extension. +TEST(RtpDependencyDescriptorExtensionTest, + TemplateMatchingSkipsInactiveChains) { + uint8_t buffer[3]; + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.templates = { + FrameDependencyTemplate().Dtis("SR").ChainDiffs({2, 2})}; + DependencyDescriptor descriptor; + descriptor.frame_dependencies = structure.templates[0]; + + // Set only 1st chain as active. + std::bitset<32> active_chains = 0b01; + descriptor.frame_dependencies.chain_diffs[1] = 1000; + + // Expect perfect template match since the only difference is for an inactive + // chain. Pefect template match consumes 3 bytes. + EXPECT_EQ(RtpDependencyDescriptorExtension::ValueSize( + structure, active_chains, descriptor), + 3u); + EXPECT_TRUE(RtpDependencyDescriptorExtension::Write( + buffer, structure, active_chains, descriptor)); +} + +TEST(RtpDependencyDescriptorExtensionTest, + AcceptsInvalidChainDiffForInactiveChainWhenChainsAreCustom) { + uint8_t buffer[256]; + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.templates = { + FrameDependencyTemplate().Dtis("SR").ChainDiffs({2, 2})}; + DependencyDescriptor descriptor; + descriptor.frame_dependencies = structure.templates[0]; + + // Set only 1st chain as active. + std::bitset<32> active_chains = 0b01; + // Set chain_diff different to the template to make it custom. + descriptor.frame_dependencies.chain_diffs[0] = 1; + // Set chain diff for inactive chain beyound limit of 255 max chain diff. + descriptor.frame_dependencies.chain_diffs[1] = 1000; + + // Because chains are custom, should use more than base 3 bytes. + EXPECT_GT(RtpDependencyDescriptorExtension::ValueSize( + structure, active_chains, descriptor), + 3u); + EXPECT_TRUE(RtpDependencyDescriptorExtension::Write( + buffer, structure, active_chains, descriptor)); +} + +} // namespace +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_reader.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_reader.cc index 07b6a3b3c32..01b893a94e8 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_reader.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_reader.cc @@ -18,13 +18,6 @@ #include "rtc_base/checks.h" namespace webrtc { -namespace { - -constexpr int kMaxTemporalId = 7; -constexpr int kMaxSpatialId = 3; -constexpr int kMaxTemplates = 64; - -} // namespace RtpDependencyDescriptorReader::RtpDependencyDescriptorReader( rtc::ArrayView<const uint8_t> raw_data, @@ -95,7 +88,7 @@ void RtpDependencyDescriptorReader::ReadTemplateLayers() { int spatial_id = 0; NextLayerIdc next_layer_idc; do { - if (templates.size() == kMaxTemplates) { + if (templates.size() == DependencyDescriptor::kMaxTemplates) { parsing_failed_ = true; break; } @@ -107,14 +100,14 @@ void RtpDependencyDescriptorReader::ReadTemplateLayers() { next_layer_idc = static_cast<NextLayerIdc>(ReadBits(2)); if (next_layer_idc == kNextTemporalLayer) { temporal_id++; - if (temporal_id > kMaxTemporalId) { + if (temporal_id >= DependencyDescriptor::kMaxTemporalIds) { parsing_failed_ = true; break; } } else if (next_layer_idc == kNextSpatialLayer) { temporal_id = 0; spatial_id++; - if (spatial_id > kMaxSpatialId) { + if (spatial_id >= DependencyDescriptor::kMaxSpatialIds) { parsing_failed_ = true; break; } @@ -198,9 +191,10 @@ void RtpDependencyDescriptorReader::ReadExtendedFields() { } void RtpDependencyDescriptorReader::ReadFrameDependencyDefinition() { - size_t template_index = (frame_dependency_template_id_ + kMaxTemplates - - structure_->structure_id) % - kMaxTemplates; + size_t template_index = + (frame_dependency_template_id_ + DependencyDescriptor::kMaxTemplates - + structure_->structure_id) % + DependencyDescriptor::kMaxTemplates; if (template_index >= structure_->templates.size()) { parsing_failed_ = true; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.cc index 9e1a4256662..c5f229c59f2 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.cc @@ -9,6 +9,7 @@ */ #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.h" +#include <bitset> #include <cstddef> #include <cstdint> #include <iterator> @@ -23,8 +24,6 @@ namespace webrtc { namespace { -constexpr int kMaxTemplates = 64; - enum class NextLayerIdc : uint64_t { kSameLayer = 0, kNextTemporal = 1, @@ -35,12 +34,8 @@ enum class NextLayerIdc : uint64_t { NextLayerIdc GetNextLayerIdc(const FrameDependencyTemplate& previous, const FrameDependencyTemplate& next) { - // TODO(danilchap): Move these constants to header shared between reader and - // writer. - static constexpr int kMaxSpatialId = 3; - static constexpr int kMaxTemporalId = 7; - RTC_DCHECK_LE(next.spatial_id, kMaxSpatialId); - RTC_DCHECK_LE(next.temporal_id, kMaxTemporalId); + RTC_DCHECK_LT(next.spatial_id, DependencyDescriptor::kMaxSpatialIds); + RTC_DCHECK_LT(next.temporal_id, DependencyDescriptor::kMaxTemporalIds); if (next.spatial_id == previous.spatial_id && next.temporal_id == previous.temporal_id) { @@ -61,9 +56,11 @@ NextLayerIdc GetNextLayerIdc(const FrameDependencyTemplate& previous, RtpDependencyDescriptorWriter::RtpDependencyDescriptorWriter( rtc::ArrayView<uint8_t> data, const FrameDependencyStructure& structure, + std::bitset<32> active_chains, const DependencyDescriptor& descriptor) : descriptor_(descriptor), structure_(structure), + active_chains_(active_chains), bit_writer_(data.data(), data.size()) { FindBestTemplate(); } @@ -74,6 +71,14 @@ bool RtpDependencyDescriptorWriter::Write() { WriteExtendedFields(); WriteFrameDependencyDefinition(); } + size_t remaining_bits = bit_writer_.RemainingBitCount(); + // Zero remaining memory to avoid leaving it uninitialized. + if (remaining_bits % 64 != 0) { + WriteBits(/*val=*/0, remaining_bits % 64); + } + for (size_t i = 0; i < remaining_bits / 64; ++i) { + WriteBits(/*val=*/0, 64); + } return !build_failed_; } @@ -126,8 +131,14 @@ RtpDependencyDescriptorWriter::CalculateMatch( result.need_custom_dtis = descriptor_.frame_dependencies.decode_target_indications != frame_template->decode_target_indications; - result.need_custom_chains = - descriptor_.frame_dependencies.chain_diffs != frame_template->chain_diffs; + result.need_custom_chains = false; + for (int i = 0; i < structure_.num_chains; ++i) { + if (active_chains_[i] && descriptor_.frame_dependencies.chain_diffs[i] != + frame_template->chain_diffs[i]) { + result.need_custom_chains = true; + break; + } + } result.extra_size_bits = 0; if (result.need_custom_fdiffs) { @@ -193,7 +204,7 @@ bool RtpDependencyDescriptorWriter::HasExtendedFields() const { uint64_t RtpDependencyDescriptorWriter::TemplateId() const { return (best_template_.template_position - structure_.templates.begin() + structure_.structure_id) % - kMaxTemplates; + DependencyDescriptor::kMaxTemplates; } void RtpDependencyDescriptorWriter::WriteBits(uint64_t val, size_t bit_count) { @@ -209,9 +220,10 @@ void RtpDependencyDescriptorWriter::WriteNonSymmetric(uint32_t value, void RtpDependencyDescriptorWriter::WriteTemplateDependencyStructure() { RTC_DCHECK_GE(structure_.structure_id, 0); - RTC_DCHECK_LT(structure_.structure_id, kMaxTemplates); + RTC_DCHECK_LT(structure_.structure_id, DependencyDescriptor::kMaxTemplates); RTC_DCHECK_GT(structure_.num_decode_targets, 0); - RTC_DCHECK_LE(structure_.num_decode_targets, 1 << 5); + RTC_DCHECK_LE(structure_.num_decode_targets, + DependencyDescriptor::kMaxDecodeTargets); WriteBits(structure_.structure_id, 6); WriteBits(structure_.num_decode_targets - 1, 5); @@ -228,7 +240,7 @@ void RtpDependencyDescriptorWriter::WriteTemplateDependencyStructure() { void RtpDependencyDescriptorWriter::WriteTemplateLayers() { const auto& templates = structure_.templates; RTC_DCHECK(!templates.empty()); - RTC_DCHECK_LE(templates.size(), kMaxTemplates); + RTC_DCHECK_LE(templates.size(), DependencyDescriptor::kMaxTemplates); RTC_DCHECK_EQ(templates[0].spatial_id, 0); RTC_DCHECK_EQ(templates[0].temporal_id, 0); @@ -363,7 +375,9 @@ void RtpDependencyDescriptorWriter::WriteFrameFdiffs() { void RtpDependencyDescriptorWriter::WriteFrameChains() { RTC_DCHECK_EQ(descriptor_.frame_dependencies.chain_diffs.size(), structure_.num_chains); - for (int chain_diff : descriptor_.frame_dependencies.chain_diffs) { + for (int i = 0; i < structure_.num_chains; ++i) { + int chain_diff = + active_chains_[i] ? descriptor_.frame_dependencies.chain_diffs[i] : 0; RTC_DCHECK_GE(chain_diff, 0); RTC_DCHECK_LT(chain_diff, 1 << 8); WriteBits(chain_diff, 8); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.h index 5a823b6e86a..99fefecea6e 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_dependency_descriptor_writer.h @@ -10,6 +10,7 @@ #ifndef MODULES_RTP_RTCP_SOURCE_RTP_DEPENDENCY_DESCRIPTOR_WRITER_H_ #define MODULES_RTP_RTCP_SOURCE_RTP_DEPENDENCY_DESCRIPTOR_WRITER_H_ +#include <bitset> #include <cstddef> #include <cstdint> #include <vector> @@ -25,6 +26,7 @@ class RtpDependencyDescriptorWriter { // |descriptor| matches the |structure|. RtpDependencyDescriptorWriter(rtc::ArrayView<uint8_t> data, const FrameDependencyStructure& structure, + std::bitset<32> active_chains, const DependencyDescriptor& descriptor); // Serializes DependencyDescriptor rtp header extension. @@ -77,6 +79,7 @@ class RtpDependencyDescriptorWriter { bool build_failed_ = false; const DependencyDescriptor& descriptor_; const FrameDependencyStructure& structure_; + std::bitset<32> active_chains_; rtc::BitBufferWriter bit_writer_; TemplateMatch best_template_; }; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc index f59f9c4ebbd..63562c5b890 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -41,7 +41,6 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo<PlayoutDelayLimits>(), CreateExtensionInfo<VideoContentTypeExtension>(), CreateExtensionInfo<VideoTimingExtension>(), - CreateExtensionInfo<FrameMarkingExtension>(), CreateExtensionInfo<RtpStreamId>(), CreateExtensionInfo<RepairedRtpStreamId>(), CreateExtensionInfo<RtpMid>(), diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc index fefe6c618f6..527874d785c 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -525,86 +525,6 @@ bool VideoTimingExtension::Write(rtc::ArrayView<uint8_t> data, return true; } -// Frame Marking. -// -// Meta-information about an RTP stream outside the encrypted media payload, -// useful for an RTP switch to do codec-agnostic selective forwarding -// without decrypting the payload. -// -// For non-scalable streams: -// 0 1 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | ID | L = 0 |S|E|I|D|0 0 0 0| -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// -// For scalable streams: -// 0 1 2 3 -// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | ID | L = 2 |S|E|I|D|B| TID | LID | TL0PICIDX | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -constexpr RTPExtensionType FrameMarkingExtension::kId; -constexpr const char FrameMarkingExtension::kUri[]; - -bool FrameMarkingExtension::IsScalable(uint8_t temporal_id, uint8_t layer_id) { - return temporal_id != kNoTemporalIdx || layer_id != kNoSpatialIdx; -} - -bool FrameMarkingExtension::Parse(rtc::ArrayView<const uint8_t> data, - FrameMarking* frame_marking) { - RTC_DCHECK(frame_marking); - - if (data.size() != 1 && data.size() != 3) - return false; - - frame_marking->start_of_frame = (data[0] & 0x80) != 0; - frame_marking->end_of_frame = (data[0] & 0x40) != 0; - frame_marking->independent_frame = (data[0] & 0x20) != 0; - frame_marking->discardable_frame = (data[0] & 0x10) != 0; - - if (data.size() == 3) { - frame_marking->base_layer_sync = (data[0] & 0x08) != 0; - frame_marking->temporal_id = data[0] & 0x7; - frame_marking->layer_id = data[1]; - frame_marking->tl0_pic_idx = data[2]; - } else { - // non-scalable - frame_marking->base_layer_sync = false; - frame_marking->temporal_id = kNoTemporalIdx; - frame_marking->layer_id = kNoSpatialIdx; - frame_marking->tl0_pic_idx = 0; - } - return true; -} - -size_t FrameMarkingExtension::ValueSize(const FrameMarking& frame_marking) { - if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) - return 3; - else - return 1; -} - -bool FrameMarkingExtension::Write(rtc::ArrayView<uint8_t> data, - const FrameMarking& frame_marking) { - RTC_DCHECK_GE(data.size(), 1); - RTC_CHECK_LE(frame_marking.temporal_id, 0x07); - data[0] = frame_marking.start_of_frame ? 0x80 : 0x00; - data[0] |= frame_marking.end_of_frame ? 0x40 : 0x00; - data[0] |= frame_marking.independent_frame ? 0x20 : 0x00; - data[0] |= frame_marking.discardable_frame ? 0x10 : 0x00; - - if (IsScalable(frame_marking.temporal_id, frame_marking.layer_id)) { - RTC_DCHECK_EQ(data.size(), 3); - data[0] |= frame_marking.base_layer_sync ? 0x08 : 0x00; - data[0] |= frame_marking.temporal_id & 0x07; - data[1] = frame_marking.layer_id; - data[2] = frame_marking.tl0_pic_idx; - } - return true; -} - // Color space including HDR metadata as an optional field. // // RTP header extension to carry color space information and optionally HDR diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h index f4517bb513e..8a81280f7b7 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -19,7 +19,6 @@ #include "api/rtp_headers.h" #include "api/video/color_space.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" @@ -217,23 +216,6 @@ class VideoTimingExtension { uint8_t offset); }; -class FrameMarkingExtension { - public: - using value_type = FrameMarking; - static constexpr RTPExtensionType kId = kRtpExtensionFrameMarking; - static constexpr const char kUri[] = - "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"; - - static bool Parse(rtc::ArrayView<const uint8_t> data, - FrameMarking* frame_marking); - static size_t ValueSize(const FrameMarking& frame_marking); - static bool Write(rtc::ArrayView<uint8_t> data, - const FrameMarking& frame_marking); - - private: - static bool IsScalable(uint8_t temporal_id, uint8_t layer_id); -}; - class ColorSpaceExtension { public: using value_type = ColorSpace; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet.cc index e054bb8306a..767c9a06883 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet.cc @@ -188,7 +188,6 @@ void RtpPacket::ZeroMutableExtensions() { case RTPExtensionType::kRtpExtensionAudioLevel: case RTPExtensionType::kRtpExtensionAbsoluteCaptureTime: case RTPExtensionType::kRtpExtensionColorSpace: - case RTPExtensionType::kRtpExtensionFrameMarking: case RTPExtensionType::kRtpExtensionGenericFrameDescriptor00: case RTPExtensionType::kRtpExtensionGenericFrameDescriptor02: case RTPExtensionType::kRtpExtensionMid: diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet_received.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet_received.cc index 56aea8eb5ec..feadee1db10 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet_received.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_packet_received.cc @@ -69,8 +69,6 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const { &header->extension.videoContentType); header->extension.has_video_timing = GetExtension<VideoTimingExtension>(&header->extension.video_timing); - header->extension.has_frame_marking = - GetExtension<FrameMarkingExtension>(&header->extension.frame_marking); GetExtension<RtpStreamId>(&header->extension.stream_id); GetExtension<RepairedRtpStreamId>(&header->extension.repaired_stream_id); GetExtension<RtpMid>(&header->extension.mid); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index fb6f8a3f8f8..f372dbe0cdf 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -39,7 +39,7 @@ const int64_t kDefaultExpectedRetransmissionTimeMs = 125; } // namespace ModuleRtpRtcpImpl::RtpSenderContext::RtpSenderContext( - const RtpRtcp::Configuration& config) + const RtpRtcpInterface::Configuration& config) : packet_history(config.clock, config.enable_rtx_padding_prioritization), packet_sender(config, &packet_history), non_paced_sender(&packet_sender), @@ -48,11 +48,11 @@ ModuleRtpRtcpImpl::RtpSenderContext::RtpSenderContext( &packet_history, config.paced_sender ? config.paced_sender : &non_paced_sender) {} -RtpRtcp::Configuration::Configuration() = default; -RtpRtcp::Configuration::Configuration(Configuration&& rhs) = default; - -std::unique_ptr<RtpRtcp> RtpRtcp::Create(const Configuration& configuration) { +std::unique_ptr<RtpRtcp> RtpRtcp::DEPRECATED_Create( + const Configuration& configuration) { RTC_DCHECK(configuration.clock); + RTC_LOG(LS_ERROR) + << "*********** USING WebRTC INTERNAL IMPLEMENTATION DETAILS ***********"; return std::make_unique<ModuleRtpRtcpImpl>(configuration); } @@ -96,23 +96,34 @@ int64_t ModuleRtpRtcpImpl::TimeUntilNextProcess() { // Process any pending tasks such as timeouts (non time critical events). void ModuleRtpRtcpImpl::Process() { const int64_t now = clock_->TimeInMilliseconds(); + // TODO(bugs.webrtc.org/11581): Figure out why we need to call Process() 200 + // times a second. next_process_time_ = now + kRtpRtcpMaxIdleTimeProcessMs; if (rtp_sender_) { if (now >= last_bitrate_process_time_ + kRtpRtcpBitrateProcessTimeMs) { rtp_sender_->packet_sender.ProcessBitrateAndNotifyObservers(); last_bitrate_process_time_ = now; + // TODO(bugs.webrtc.org/11581): Is this a bug? At the top of the function, + // next_process_time_ is incremented by 5ms, here we effectively do a + // std::min() of (now + 5ms, now + 10ms). Seems like this is a no-op? next_process_time_ = std::min(next_process_time_, now + kRtpRtcpBitrateProcessTimeMs); } } + // TODO(bugs.webrtc.org/11581): We update the RTT once a second, whereas other + // things that run in this method are updated much more frequently. Move the + // RTT checking over to the worker thread, which matches better with where the + // stats are maintained. bool process_rtt = now >= last_rtt_process_time_ + kRtpRtcpRttProcessTimeMs; if (rtcp_sender_.Sending()) { // Process RTT if we have received a report block and we haven't // processed RTT for at least |kRtpRtcpRttProcessTimeMs| milliseconds. - if (rtcp_receiver_.LastReceivedReportBlockMs() > last_rtt_process_time_ && - process_rtt) { + // Note that LastReceivedReportBlockMs() grabs a lock, so check + // |process_rtt| first. + if (process_rtt && + rtcp_receiver_.LastReceivedReportBlockMs() > last_rtt_process_time_) { std::vector<RTCPReportBlock> receive_blocks; rtcp_receiver_.StatisticsReceived(&receive_blocks); int64_t max_rtt = 0; @@ -129,6 +140,12 @@ void ModuleRtpRtcpImpl::Process() { // Verify receiver reports are delivered and the reported sequence number // is increasing. + // TODO(bugs.webrtc.org/11581): The timeout value needs to be checked every + // few seconds (see internals of RtcpRrTimeout). Here, we may be polling it + // a couple of hundred times a second, which isn't great since it grabs a + // lock. Note also that LastReceivedReportBlockMs() (called above) and + // RtcpRrTimeout() both grab the same lock and check the same timer, so + // it should be possible to consolidate that work somehow. if (rtcp_receiver_.RtcpRrTimeout()) { RTC_LOG_F(LS_WARNING) << "Timeout: No RTCP RR received."; } else if (rtcp_receiver_.RtcpRrSequenceNumberTimeout()) { @@ -159,6 +176,9 @@ void ModuleRtpRtcpImpl::Process() { // Get processed rtt. if (process_rtt) { last_rtt_process_time_ = now; + // TODO(bugs.webrtc.org/11581): Is this a bug? At the top of the function, + // next_process_time_ is incremented by 5ms, here we effectively do a + // std::min() of (now + 5ms, now + 1000ms). Seems like this is a no-op? next_process_time_ = std::min( next_process_time_, last_rtt_process_time_ + kRtpRtcpRttProcessTimeMs); if (rtt_stats_) { @@ -292,7 +312,7 @@ RTCPSender::FeedbackState ModuleRtpRtcpImpl::GetFeedbackState() { state.send_bitrate = rtp_sender_->packet_sender.GetSendRates().Sum().bps<uint32_t>(); } - state.module = this; + state.receiver = &rtcp_receiver_; LastReceivedNTP(&state.last_rr_ntp_secs, &state.last_rr_ntp_frac, &state.remote_sr); @@ -503,7 +523,8 @@ int32_t ModuleRtpRtcpImpl::SetRTCPApplicationSpecificData( const uint32_t name, const uint8_t* data, const uint16_t length) { - return rtcp_sender_.SetApplicationSpecificData(sub_type, name, data, length); + RTC_NOTREACHED() << "Not implemented"; + return -1; } void ModuleRtpRtcpImpl::SetRtcpXrRrtrStatus(bool enable) { @@ -773,11 +794,6 @@ bool ModuleRtpRtcpImpl::LastReceivedNTP( return true; } -// Called from RTCPsender. -std::vector<rtcp::TmmbItem> ModuleRtpRtcpImpl::BoundingSet(bool* tmmbr_owner) { - return rtcp_receiver_.BoundingSet(tmmbr_owner); -} - void ModuleRtpRtcpImpl::set_rtt_ms(int64_t rtt_ms) { { rtc::CritScope cs(&critical_section_rtt_); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h index debb433297f..989b8d37176 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -26,13 +26,13 @@ #include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" // RTCPPacketType +#include "modules/rtp_rtcp/source/deprecated/deprecated_rtp_sender_egress.h" #include "modules/rtp_rtcp/source/rtcp_packet/tmmb_item.h" #include "modules/rtp_rtcp/source/rtcp_receiver.h" #include "modules/rtp_rtcp/source/rtcp_sender.h" #include "modules/rtp_rtcp/source/rtp_packet_history.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" #include "modules/rtp_rtcp/source/rtp_sender.h" -#include "modules/rtp_rtcp/source/rtp_sender_egress.h" #include "rtc_base/critical_section.h" #include "rtc_base/gtest_prod_util.h" @@ -42,9 +42,11 @@ class Clock; struct PacedPacketInfo; struct RTPVideoHeader; +// DEPRECATED. class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { public: - explicit ModuleRtpRtcpImpl(const RtpRtcp::Configuration& configuration); + explicit ModuleRtpRtcpImpl( + const RtpRtcpInterface::Configuration& configuration); ~ModuleRtpRtcpImpl() override; // Returns the number of milliseconds until the module want a worker thread to @@ -257,8 +259,6 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { uint32_t* NTPfrac, uint32_t* remote_sr) const; - std::vector<rtcp::TmmbItem> BoundingSet(bool* tmmbr_owner); - void BitrateSent(uint32_t* total_rate, uint32_t* video_rate, uint32_t* fec_rate, @@ -294,6 +294,10 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { RTCPReceiver* rtcp_receiver() { return &rtcp_receiver_; } const RTCPReceiver* rtcp_receiver() const { return &rtcp_receiver_; } + void SetMediaHasBeenSent(bool media_has_been_sent) { + rtp_sender_->packet_sender.SetMediaHasBeenSent(media_has_been_sent); + } + Clock* clock() const { return clock_; } // TODO(sprang): Remove when usage is gone. @@ -305,14 +309,14 @@ class ModuleRtpRtcpImpl : public RtpRtcp, public RTCPReceiver::ModuleRtpRtcp { FRIEND_TEST_ALL_PREFIXES(RtpRtcpImplTest, RttForReceiverOnly); struct RtpSenderContext { - explicit RtpSenderContext(const RtpRtcp::Configuration& config); + explicit RtpSenderContext(const RtpRtcpInterface::Configuration& config); // Storage of packets, for retransmissions and padding, if applicable. RtpPacketHistory packet_history; // Handles final time timestamping/stats/etc and handover to Transport. - RtpSenderEgress packet_sender; + DEPRECATED_RtpSenderEgress packet_sender; // If no paced sender configured, this class will be used to pass packets // from |packet_generator_| to |packet_sender_|. - RtpSenderEgress::NonPacedPacketSender non_paced_sender; + DEPRECATED_RtpSenderEgress::NonPacedPacketSender non_paced_sender; // Handles creation of RTP packets to be sent. RTPSender packet_generator; }; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc new file mode 100644 index 00000000000..e50f72bb292 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc @@ -0,0 +1,769 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" + +#include <string.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <set> +#include <string> +#include <utility> + +#include "api/transport/field_trial_based_config.h" +#include "modules/rtp_rtcp/source/rtcp_packet/dlrr.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_config.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +#ifdef _WIN32 +// Disable warning C4355: 'this' : used in base member initializer list. +#pragma warning(disable : 4355) +#endif + +namespace webrtc { +namespace { +const int64_t kRtpRtcpMaxIdleTimeProcessMs = 5; +const int64_t kRtpRtcpRttProcessTimeMs = 1000; +const int64_t kRtpRtcpBitrateProcessTimeMs = 10; +const int64_t kDefaultExpectedRetransmissionTimeMs = 125; +} // namespace + +ModuleRtpRtcpImpl2::RtpSenderContext::RtpSenderContext( + const RtpRtcpInterface::Configuration& config) + : packet_history(config.clock, config.enable_rtx_padding_prioritization), + packet_sender(config, &packet_history), + non_paced_sender(&packet_sender), + packet_generator( + config, + &packet_history, + config.paced_sender ? config.paced_sender : &non_paced_sender) {} + +ModuleRtpRtcpImpl2::ModuleRtpRtcpImpl2(const Configuration& configuration) + : rtcp_sender_(configuration), + rtcp_receiver_(configuration, this), + clock_(configuration.clock), + last_bitrate_process_time_(clock_->TimeInMilliseconds()), + last_rtt_process_time_(clock_->TimeInMilliseconds()), + next_process_time_(clock_->TimeInMilliseconds() + + kRtpRtcpMaxIdleTimeProcessMs), + packet_overhead_(28), // IPV4 UDP. + nack_last_time_sent_full_ms_(0), + nack_last_seq_number_sent_(0), + remote_bitrate_(configuration.remote_bitrate_estimator), + rtt_stats_(configuration.rtt_stats), + rtt_ms_(0) { + process_thread_checker_.Detach(); + if (!configuration.receiver_only) { + rtp_sender_ = std::make_unique<RtpSenderContext>(configuration); + // Make sure rtcp sender use same timestamp offset as rtp sender. + rtcp_sender_.SetTimestampOffset( + rtp_sender_->packet_generator.TimestampOffset()); + } + + // Set default packet size limit. + // TODO(nisse): Kind-of duplicates + // webrtc::VideoSendStream::Config::Rtp::kDefaultMaxPacketSize. + const size_t kTcpOverIpv4HeaderSize = 40; + SetMaxRtpPacketSize(IP_PACKET_SIZE - kTcpOverIpv4HeaderSize); +} + +ModuleRtpRtcpImpl2::~ModuleRtpRtcpImpl2() { + RTC_DCHECK_RUN_ON(&construction_thread_checker_); +} + +// static +std::unique_ptr<ModuleRtpRtcpImpl2> ModuleRtpRtcpImpl2::Create( + const Configuration& configuration) { + RTC_DCHECK(configuration.clock); + RTC_DCHECK(TaskQueueBase::Current()); + return std::make_unique<ModuleRtpRtcpImpl2>(configuration); +} + +// Returns the number of milliseconds until the module want a worker thread +// to call Process. +int64_t ModuleRtpRtcpImpl2::TimeUntilNextProcess() { + RTC_DCHECK_RUN_ON(&process_thread_checker_); + return std::max<int64_t>(0, + next_process_time_ - clock_->TimeInMilliseconds()); +} + +// Process any pending tasks such as timeouts (non time critical events). +void ModuleRtpRtcpImpl2::Process() { + RTC_DCHECK_RUN_ON(&process_thread_checker_); + const int64_t now = clock_->TimeInMilliseconds(); + // TODO(bugs.webrtc.org/11581): Figure out why we need to call Process() 200 + // times a second. + next_process_time_ = now + kRtpRtcpMaxIdleTimeProcessMs; + + if (rtp_sender_) { + if (now >= last_bitrate_process_time_ + kRtpRtcpBitrateProcessTimeMs) { + rtp_sender_->packet_sender.ProcessBitrateAndNotifyObservers(); + last_bitrate_process_time_ = now; + // TODO(bugs.webrtc.org/11581): Is this a bug? At the top of the function, + // next_process_time_ is incremented by 5ms, here we effectively do a + // std::min() of (now + 5ms, now + 10ms). Seems like this is a no-op? + next_process_time_ = + std::min(next_process_time_, now + kRtpRtcpBitrateProcessTimeMs); + } + } + + // TODO(bugs.webrtc.org/11581): We update the RTT once a second, whereas other + // things that run in this method are updated much more frequently. Move the + // RTT checking over to the worker thread, which matches better with where the + // stats are maintained. + bool process_rtt = now >= last_rtt_process_time_ + kRtpRtcpRttProcessTimeMs; + if (rtcp_sender_.Sending()) { + // Process RTT if we have received a report block and we haven't + // processed RTT for at least |kRtpRtcpRttProcessTimeMs| milliseconds. + // Note that LastReceivedReportBlockMs() grabs a lock, so check + // |process_rtt| first. + if (process_rtt && + rtcp_receiver_.LastReceivedReportBlockMs() > last_rtt_process_time_) { + std::vector<RTCPReportBlock> receive_blocks; + rtcp_receiver_.StatisticsReceived(&receive_blocks); + int64_t max_rtt = 0; + for (std::vector<RTCPReportBlock>::iterator it = receive_blocks.begin(); + it != receive_blocks.end(); ++it) { + int64_t rtt = 0; + rtcp_receiver_.RTT(it->sender_ssrc, &rtt, NULL, NULL, NULL); + max_rtt = (rtt > max_rtt) ? rtt : max_rtt; + } + // Report the rtt. + if (rtt_stats_ && max_rtt != 0) + rtt_stats_->OnRttUpdate(max_rtt); + } + + // Verify receiver reports are delivered and the reported sequence number + // is increasing. + // TODO(bugs.webrtc.org/11581): The timeout value needs to be checked every + // few seconds (see internals of RtcpRrTimeout). Here, we may be polling it + // a couple of hundred times a second, which isn't great since it grabs a + // lock. Note also that LastReceivedReportBlockMs() (called above) and + // RtcpRrTimeout() both grab the same lock and check the same timer, so + // it should be possible to consolidate that work somehow. + if (rtcp_receiver_.RtcpRrTimeout()) { + RTC_LOG_F(LS_WARNING) << "Timeout: No RTCP RR received."; + } else if (rtcp_receiver_.RtcpRrSequenceNumberTimeout()) { + RTC_LOG_F(LS_WARNING) << "Timeout: No increase in RTCP RR extended " + "highest sequence number."; + } + + if (remote_bitrate_ && rtcp_sender_.TMMBR()) { + unsigned int target_bitrate = 0; + std::vector<unsigned int> ssrcs; + if (remote_bitrate_->LatestEstimate(&ssrcs, &target_bitrate)) { + if (!ssrcs.empty()) { + target_bitrate = target_bitrate / ssrcs.size(); + } + rtcp_sender_.SetTargetBitrate(target_bitrate); + } + } + } else { + // Report rtt from receiver. + if (process_rtt) { + int64_t rtt_ms; + if (rtt_stats_ && rtcp_receiver_.GetAndResetXrRrRtt(&rtt_ms)) { + rtt_stats_->OnRttUpdate(rtt_ms); + } + } + } + + // Get processed rtt. + if (process_rtt) { + last_rtt_process_time_ = now; + // TODO(bugs.webrtc.org/11581): Is this a bug? At the top of the function, + // next_process_time_ is incremented by 5ms, here we effectively do a + // std::min() of (now + 5ms, now + 1000ms). Seems like this is a no-op? + next_process_time_ = std::min( + next_process_time_, last_rtt_process_time_ + kRtpRtcpRttProcessTimeMs); + if (rtt_stats_) { + // Make sure we have a valid RTT before setting. + int64_t last_rtt = rtt_stats_->LastProcessedRtt(); + if (last_rtt >= 0) + set_rtt_ms(last_rtt); + } + } + + if (rtcp_sender_.TimeToSendRTCPReport()) + rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); + + if (rtcp_sender_.TMMBR() && rtcp_receiver_.UpdateTmmbrTimers()) { + rtcp_receiver_.NotifyTmmbrUpdated(); + } +} + +void ModuleRtpRtcpImpl2::SetRtxSendStatus(int mode) { + rtp_sender_->packet_generator.SetRtxStatus(mode); +} + +int ModuleRtpRtcpImpl2::RtxSendStatus() const { + return rtp_sender_ ? rtp_sender_->packet_generator.RtxStatus() : kRtxOff; +} + +void ModuleRtpRtcpImpl2::SetRtxSendPayloadType(int payload_type, + int associated_payload_type) { + rtp_sender_->packet_generator.SetRtxPayloadType(payload_type, + associated_payload_type); +} + +absl::optional<uint32_t> ModuleRtpRtcpImpl2::RtxSsrc() const { + return rtp_sender_ ? rtp_sender_->packet_generator.RtxSsrc() : absl::nullopt; +} + +absl::optional<uint32_t> ModuleRtpRtcpImpl2::FlexfecSsrc() const { + if (rtp_sender_) { + return rtp_sender_->packet_generator.FlexfecSsrc(); + } + return absl::nullopt; +} + +void ModuleRtpRtcpImpl2::IncomingRtcpPacket(const uint8_t* rtcp_packet, + const size_t length) { + rtcp_receiver_.IncomingPacket(rtcp_packet, length); +} + +void ModuleRtpRtcpImpl2::RegisterSendPayloadFrequency(int payload_type, + int payload_frequency) { + rtcp_sender_.SetRtpClockRate(payload_type, payload_frequency); +} + +int32_t ModuleRtpRtcpImpl2::DeRegisterSendPayload(const int8_t payload_type) { + return 0; +} + +uint32_t ModuleRtpRtcpImpl2::StartTimestamp() const { + return rtp_sender_->packet_generator.TimestampOffset(); +} + +// Configure start timestamp, default is a random number. +void ModuleRtpRtcpImpl2::SetStartTimestamp(const uint32_t timestamp) { + rtcp_sender_.SetTimestampOffset(timestamp); + rtp_sender_->packet_generator.SetTimestampOffset(timestamp); + rtp_sender_->packet_sender.SetTimestampOffset(timestamp); +} + +uint16_t ModuleRtpRtcpImpl2::SequenceNumber() const { + return rtp_sender_->packet_generator.SequenceNumber(); +} + +// Set SequenceNumber, default is a random number. +void ModuleRtpRtcpImpl2::SetSequenceNumber(const uint16_t seq_num) { + rtp_sender_->packet_generator.SetSequenceNumber(seq_num); +} + +void ModuleRtpRtcpImpl2::SetRtpState(const RtpState& rtp_state) { + rtp_sender_->packet_generator.SetRtpState(rtp_state); + rtp_sender_->packet_sender.SetMediaHasBeenSent(rtp_state.media_has_been_sent); + rtcp_sender_.SetTimestampOffset(rtp_state.start_timestamp); +} + +void ModuleRtpRtcpImpl2::SetRtxState(const RtpState& rtp_state) { + rtp_sender_->packet_generator.SetRtxRtpState(rtp_state); +} + +RtpState ModuleRtpRtcpImpl2::GetRtpState() const { + RtpState state = rtp_sender_->packet_generator.GetRtpState(); + state.media_has_been_sent = rtp_sender_->packet_sender.MediaHasBeenSent(); + return state; +} + +RtpState ModuleRtpRtcpImpl2::GetRtxState() const { + return rtp_sender_->packet_generator.GetRtxRtpState(); +} + +void ModuleRtpRtcpImpl2::SetRid(const std::string& rid) { + if (rtp_sender_) { + rtp_sender_->packet_generator.SetRid(rid); + } +} + +void ModuleRtpRtcpImpl2::SetMid(const std::string& mid) { + if (rtp_sender_) { + rtp_sender_->packet_generator.SetMid(mid); + } + // TODO(bugs.webrtc.org/4050): If we end up supporting the MID SDES item for + // RTCP, this will need to be passed down to the RTCPSender also. +} + +void ModuleRtpRtcpImpl2::SetCsrcs(const std::vector<uint32_t>& csrcs) { + rtcp_sender_.SetCsrcs(csrcs); + rtp_sender_->packet_generator.SetCsrcs(csrcs); +} + +// TODO(pbos): Handle media and RTX streams separately (separate RTCP +// feedbacks). +RTCPSender::FeedbackState ModuleRtpRtcpImpl2::GetFeedbackState() { + RTCPSender::FeedbackState state; + // This is called also when receiver_only is true. Hence below + // checks that rtp_sender_ exists. + if (rtp_sender_) { + StreamDataCounters rtp_stats; + StreamDataCounters rtx_stats; + rtp_sender_->packet_sender.GetDataCounters(&rtp_stats, &rtx_stats); + state.packets_sent = + rtp_stats.transmitted.packets + rtx_stats.transmitted.packets; + state.media_bytes_sent = rtp_stats.transmitted.payload_bytes + + rtx_stats.transmitted.payload_bytes; + state.send_bitrate = + rtp_sender_->packet_sender.GetSendRates().Sum().bps<uint32_t>(); + } + state.receiver = &rtcp_receiver_; + + LastReceivedNTP(&state.last_rr_ntp_secs, &state.last_rr_ntp_frac, + &state.remote_sr); + + state.last_xr_rtis = rtcp_receiver_.ConsumeReceivedXrReferenceTimeInfo(); + + return state; +} + +// TODO(nisse): This method shouldn't be called for a receive-only +// stream. Delete rtp_sender_ check as soon as all applications are +// updated. +int32_t ModuleRtpRtcpImpl2::SetSendingStatus(const bool sending) { + if (rtcp_sender_.Sending() != sending) { + // Sends RTCP BYE when going from true to false + if (rtcp_sender_.SetSendingStatus(GetFeedbackState(), sending) != 0) { + RTC_LOG(LS_WARNING) << "Failed to send RTCP BYE"; + } + } + return 0; +} + +bool ModuleRtpRtcpImpl2::Sending() const { + return rtcp_sender_.Sending(); +} + +// TODO(nisse): This method shouldn't be called for a receive-only +// stream. Delete rtp_sender_ check as soon as all applications are +// updated. +void ModuleRtpRtcpImpl2::SetSendingMediaStatus(const bool sending) { + if (rtp_sender_) { + rtp_sender_->packet_generator.SetSendingMediaStatus(sending); + } else { + RTC_DCHECK(!sending); + } +} + +bool ModuleRtpRtcpImpl2::SendingMedia() const { + return rtp_sender_ ? rtp_sender_->packet_generator.SendingMedia() : false; +} + +bool ModuleRtpRtcpImpl2::IsAudioConfigured() const { + return rtp_sender_ ? rtp_sender_->packet_generator.IsAudioConfigured() + : false; +} + +void ModuleRtpRtcpImpl2::SetAsPartOfAllocation(bool part_of_allocation) { + RTC_CHECK(rtp_sender_); + rtp_sender_->packet_sender.ForceIncludeSendPacketsInAllocation( + part_of_allocation); +} + +bool ModuleRtpRtcpImpl2::OnSendingRtpFrame(uint32_t timestamp, + int64_t capture_time_ms, + int payload_type, + bool force_sender_report) { + if (!Sending()) + return false; + + rtcp_sender_.SetLastRtpTime(timestamp, capture_time_ms, payload_type); + // Make sure an RTCP report isn't queued behind a key frame. + if (rtcp_sender_.TimeToSendRTCPReport(force_sender_report)) + rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpReport); + + return true; +} + +bool ModuleRtpRtcpImpl2::TrySendPacket(RtpPacketToSend* packet, + const PacedPacketInfo& pacing_info) { + RTC_DCHECK(rtp_sender_); + // TODO(sprang): Consider if we can remove this check. + if (!rtp_sender_->packet_generator.SendingMedia()) { + return false; + } + rtp_sender_->packet_sender.SendPacket(packet, pacing_info); + return true; +} + +void ModuleRtpRtcpImpl2::OnPacketsAcknowledged( + rtc::ArrayView<const uint16_t> sequence_numbers) { + RTC_DCHECK(rtp_sender_); + rtp_sender_->packet_history.CullAcknowledgedPackets(sequence_numbers); +} + +bool ModuleRtpRtcpImpl2::SupportsPadding() const { + RTC_DCHECK(rtp_sender_); + return rtp_sender_->packet_generator.SupportsPadding(); +} + +bool ModuleRtpRtcpImpl2::SupportsRtxPayloadPadding() const { + RTC_DCHECK(rtp_sender_); + return rtp_sender_->packet_generator.SupportsRtxPayloadPadding(); +} + +std::vector<std::unique_ptr<RtpPacketToSend>> +ModuleRtpRtcpImpl2::GeneratePadding(size_t target_size_bytes) { + RTC_DCHECK(rtp_sender_); + return rtp_sender_->packet_generator.GeneratePadding( + target_size_bytes, rtp_sender_->packet_sender.MediaHasBeenSent()); +} + +std::vector<RtpSequenceNumberMap::Info> +ModuleRtpRtcpImpl2::GetSentRtpPacketInfos( + rtc::ArrayView<const uint16_t> sequence_numbers) const { + RTC_DCHECK(rtp_sender_); + return rtp_sender_->packet_sender.GetSentRtpPacketInfos(sequence_numbers); +} + +size_t ModuleRtpRtcpImpl2::ExpectedPerPacketOverhead() const { + if (!rtp_sender_) { + return 0; + } + return rtp_sender_->packet_generator.ExpectedPerPacketOverhead(); +} + +size_t ModuleRtpRtcpImpl2::MaxRtpPacketSize() const { + RTC_DCHECK(rtp_sender_); + return rtp_sender_->packet_generator.MaxRtpPacketSize(); +} + +void ModuleRtpRtcpImpl2::SetMaxRtpPacketSize(size_t rtp_packet_size) { + RTC_DCHECK_LE(rtp_packet_size, IP_PACKET_SIZE) + << "rtp packet size too large: " << rtp_packet_size; + RTC_DCHECK_GT(rtp_packet_size, packet_overhead_) + << "rtp packet size too small: " << rtp_packet_size; + + rtcp_sender_.SetMaxRtpPacketSize(rtp_packet_size); + if (rtp_sender_) { + rtp_sender_->packet_generator.SetMaxRtpPacketSize(rtp_packet_size); + } +} + +RtcpMode ModuleRtpRtcpImpl2::RTCP() const { + return rtcp_sender_.Status(); +} + +// Configure RTCP status i.e on/off. +void ModuleRtpRtcpImpl2::SetRTCPStatus(const RtcpMode method) { + rtcp_sender_.SetRTCPStatus(method); +} + +int32_t ModuleRtpRtcpImpl2::SetCNAME(const char* c_name) { + return rtcp_sender_.SetCNAME(c_name); +} + +int32_t ModuleRtpRtcpImpl2::RemoteNTP(uint32_t* received_ntpsecs, + uint32_t* received_ntpfrac, + uint32_t* rtcp_arrival_time_secs, + uint32_t* rtcp_arrival_time_frac, + uint32_t* rtcp_timestamp) const { + return rtcp_receiver_.NTP(received_ntpsecs, received_ntpfrac, + rtcp_arrival_time_secs, rtcp_arrival_time_frac, + rtcp_timestamp) + ? 0 + : -1; +} + +// Get RoundTripTime. +int32_t ModuleRtpRtcpImpl2::RTT(const uint32_t remote_ssrc, + int64_t* rtt, + int64_t* avg_rtt, + int64_t* min_rtt, + int64_t* max_rtt) const { + int32_t ret = rtcp_receiver_.RTT(remote_ssrc, rtt, avg_rtt, min_rtt, max_rtt); + if (rtt && *rtt == 0) { + // Try to get RTT from RtcpRttStats class. + *rtt = rtt_ms(); + } + return ret; +} + +int64_t ModuleRtpRtcpImpl2::ExpectedRetransmissionTimeMs() const { + int64_t expected_retransmission_time_ms = rtt_ms(); + if (expected_retransmission_time_ms > 0) { + return expected_retransmission_time_ms; + } + // No rtt available (|kRtpRtcpRttProcessTimeMs| not yet passed?), so try to + // poll avg_rtt_ms directly from rtcp receiver. + if (rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), nullptr, + &expected_retransmission_time_ms, nullptr, + nullptr) == 0) { + return expected_retransmission_time_ms; + } + return kDefaultExpectedRetransmissionTimeMs; +} + +// Force a send of an RTCP packet. +// Normal SR and RR are triggered via the process function. +int32_t ModuleRtpRtcpImpl2::SendRTCP(RTCPPacketType packet_type) { + return rtcp_sender_.SendRTCP(GetFeedbackState(), packet_type); +} + +void ModuleRtpRtcpImpl2::SetRtcpXrRrtrStatus(bool enable) { + rtcp_receiver_.SetRtcpXrRrtrStatus(enable); + rtcp_sender_.SendRtcpXrReceiverReferenceTime(enable); +} + +bool ModuleRtpRtcpImpl2::RtcpXrRrtrStatus() const { + return rtcp_sender_.RtcpXrReceiverReferenceTime(); +} + +void ModuleRtpRtcpImpl2::GetSendStreamDataCounters( + StreamDataCounters* rtp_counters, + StreamDataCounters* rtx_counters) const { + rtp_sender_->packet_sender.GetDataCounters(rtp_counters, rtx_counters); +} + +// Received RTCP report. +int32_t ModuleRtpRtcpImpl2::RemoteRTCPStat( + std::vector<RTCPReportBlock>* receive_blocks) const { + return rtcp_receiver_.StatisticsReceived(receive_blocks); +} + +std::vector<ReportBlockData> ModuleRtpRtcpImpl2::GetLatestReportBlockData() + const { + return rtcp_receiver_.GetLatestReportBlockData(); +} + +// (REMB) Receiver Estimated Max Bitrate. +void ModuleRtpRtcpImpl2::SetRemb(int64_t bitrate_bps, + std::vector<uint32_t> ssrcs) { + rtcp_sender_.SetRemb(bitrate_bps, std::move(ssrcs)); +} + +void ModuleRtpRtcpImpl2::UnsetRemb() { + rtcp_sender_.UnsetRemb(); +} + +void ModuleRtpRtcpImpl2::SetExtmapAllowMixed(bool extmap_allow_mixed) { + rtp_sender_->packet_generator.SetExtmapAllowMixed(extmap_allow_mixed); +} + +void ModuleRtpRtcpImpl2::RegisterRtpHeaderExtension(absl::string_view uri, + int id) { + bool registered = + rtp_sender_->packet_generator.RegisterRtpHeaderExtension(uri, id); + RTC_CHECK(registered); +} + +int32_t ModuleRtpRtcpImpl2::DeregisterSendRtpHeaderExtension( + const RTPExtensionType type) { + return rtp_sender_->packet_generator.DeregisterRtpHeaderExtension(type); +} +void ModuleRtpRtcpImpl2::DeregisterSendRtpHeaderExtension( + absl::string_view uri) { + rtp_sender_->packet_generator.DeregisterRtpHeaderExtension(uri); +} + +void ModuleRtpRtcpImpl2::SetTmmbn(std::vector<rtcp::TmmbItem> bounding_set) { + rtcp_sender_.SetTmmbn(std::move(bounding_set)); +} + +// Send a Negative acknowledgment packet. +int32_t ModuleRtpRtcpImpl2::SendNACK(const uint16_t* nack_list, + const uint16_t size) { + uint16_t nack_length = size; + uint16_t start_id = 0; + int64_t now_ms = clock_->TimeInMilliseconds(); + if (TimeToSendFullNackList(now_ms)) { + nack_last_time_sent_full_ms_ = now_ms; + } else { + // Only send extended list. + if (nack_last_seq_number_sent_ == nack_list[size - 1]) { + // Last sequence number is the same, do not send list. + return 0; + } + // Send new sequence numbers. + for (int i = 0; i < size; ++i) { + if (nack_last_seq_number_sent_ == nack_list[i]) { + start_id = i + 1; + break; + } + } + nack_length = size - start_id; + } + + // Our RTCP NACK implementation is limited to kRtcpMaxNackFields sequence + // numbers per RTCP packet. + if (nack_length > kRtcpMaxNackFields) { + nack_length = kRtcpMaxNackFields; + } + nack_last_seq_number_sent_ = nack_list[start_id + nack_length - 1]; + + return rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpNack, nack_length, + &nack_list[start_id]); +} + +void ModuleRtpRtcpImpl2::SendNack( + const std::vector<uint16_t>& sequence_numbers) { + rtcp_sender_.SendRTCP(GetFeedbackState(), kRtcpNack, sequence_numbers.size(), + sequence_numbers.data()); +} + +bool ModuleRtpRtcpImpl2::TimeToSendFullNackList(int64_t now) const { + // Use RTT from RtcpRttStats class if provided. + int64_t rtt = rtt_ms(); + if (rtt == 0) { + rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), NULL, &rtt, NULL, NULL); + } + + const int64_t kStartUpRttMs = 100; + int64_t wait_time = 5 + ((rtt * 3) >> 1); // 5 + RTT * 1.5. + if (rtt == 0) { + wait_time = kStartUpRttMs; + } + + // Send a full NACK list once within every |wait_time|. + return now - nack_last_time_sent_full_ms_ > wait_time; +} + +// Store the sent packets, needed to answer to Negative acknowledgment requests. +void ModuleRtpRtcpImpl2::SetStorePacketsStatus(const bool enable, + const uint16_t number_to_store) { + rtp_sender_->packet_history.SetStorePacketsStatus( + enable ? RtpPacketHistory::StorageMode::kStoreAndCull + : RtpPacketHistory::StorageMode::kDisabled, + number_to_store); +} + +bool ModuleRtpRtcpImpl2::StorePackets() const { + return rtp_sender_->packet_history.GetStorageMode() != + RtpPacketHistory::StorageMode::kDisabled; +} + +void ModuleRtpRtcpImpl2::SendCombinedRtcpPacket( + std::vector<std::unique_ptr<rtcp::RtcpPacket>> rtcp_packets) { + rtcp_sender_.SendCombinedRtcpPacket(std::move(rtcp_packets)); +} + +int32_t ModuleRtpRtcpImpl2::SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) { + return rtcp_sender_.SendLossNotification( + GetFeedbackState(), last_decoded_seq_num, last_received_seq_num, + decodability_flag, buffering_allowed); +} + +void ModuleRtpRtcpImpl2::SetRemoteSSRC(const uint32_t ssrc) { + // Inform about the incoming SSRC. + rtcp_sender_.SetRemoteSSRC(ssrc); + rtcp_receiver_.SetRemoteSSRC(ssrc); +} + +// TODO(nisse): Delete video_rate amd fec_rate arguments. +void ModuleRtpRtcpImpl2::BitrateSent(uint32_t* total_rate, + uint32_t* video_rate, + uint32_t* fec_rate, + uint32_t* nack_rate) const { + RtpSendRates send_rates = rtp_sender_->packet_sender.GetSendRates(); + *total_rate = send_rates.Sum().bps<uint32_t>(); + if (video_rate) + *video_rate = 0; + if (fec_rate) + *fec_rate = 0; + *nack_rate = send_rates[RtpPacketMediaType::kRetransmission].bps<uint32_t>(); +} + +RtpSendRates ModuleRtpRtcpImpl2::GetSendRates() const { + return rtp_sender_->packet_sender.GetSendRates(); +} + +void ModuleRtpRtcpImpl2::OnRequestSendReport() { + SendRTCP(kRtcpSr); +} + +void ModuleRtpRtcpImpl2::OnReceivedNack( + const std::vector<uint16_t>& nack_sequence_numbers) { + if (!rtp_sender_) + return; + + if (!StorePackets() || nack_sequence_numbers.empty()) { + return; + } + // Use RTT from RtcpRttStats class if provided. + int64_t rtt = rtt_ms(); + if (rtt == 0) { + rtcp_receiver_.RTT(rtcp_receiver_.RemoteSSRC(), NULL, &rtt, NULL, NULL); + } + rtp_sender_->packet_generator.OnReceivedNack(nack_sequence_numbers, rtt); +} + +void ModuleRtpRtcpImpl2::OnReceivedRtcpReportBlocks( + const ReportBlockList& report_blocks) { + if (rtp_sender_) { + uint32_t ssrc = SSRC(); + absl::optional<uint32_t> rtx_ssrc; + if (rtp_sender_->packet_generator.RtxStatus() != kRtxOff) { + rtx_ssrc = rtp_sender_->packet_generator.RtxSsrc(); + } + + for (const RTCPReportBlock& report_block : report_blocks) { + if (ssrc == report_block.source_ssrc) { + rtp_sender_->packet_generator.OnReceivedAckOnSsrc( + report_block.extended_highest_sequence_number); + } else if (rtx_ssrc && *rtx_ssrc == report_block.source_ssrc) { + rtp_sender_->packet_generator.OnReceivedAckOnRtxSsrc( + report_block.extended_highest_sequence_number); + } + } + } +} + +bool ModuleRtpRtcpImpl2::LastReceivedNTP( + uint32_t* rtcp_arrival_time_secs, // When we got the last report. + uint32_t* rtcp_arrival_time_frac, + uint32_t* remote_sr) const { + // Remote SR: NTP inside the last received (mid 16 bits from sec and frac). + uint32_t ntp_secs = 0; + uint32_t ntp_frac = 0; + + if (!rtcp_receiver_.NTP(&ntp_secs, &ntp_frac, rtcp_arrival_time_secs, + rtcp_arrival_time_frac, NULL)) { + return false; + } + *remote_sr = + ((ntp_secs & 0x0000ffff) << 16) + ((ntp_frac & 0xffff0000) >> 16); + return true; +} + +void ModuleRtpRtcpImpl2::set_rtt_ms(int64_t rtt_ms) { + { + rtc::CritScope cs(&critical_section_rtt_); + rtt_ms_ = rtt_ms; + } + if (rtp_sender_) { + rtp_sender_->packet_history.SetRtt(rtt_ms); + } +} + +int64_t ModuleRtpRtcpImpl2::rtt_ms() const { + rtc::CritScope cs(&critical_section_rtt_); + return rtt_ms_; +} + +void ModuleRtpRtcpImpl2::SetVideoBitrateAllocation( + const VideoBitrateAllocation& bitrate) { + rtcp_sender_.SetVideoBitrateAllocation(bitrate); +} + +RTPSender* ModuleRtpRtcpImpl2::RtpSender() { + return rtp_sender_ ? &rtp_sender_->packet_generator : nullptr; +} + +const RTPSender* ModuleRtpRtcpImpl2::RtpSender() const { + return rtp_sender_ ? &rtp_sender_->packet_generator : nullptr; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.h new file mode 100644 index 00000000000..276f88a6b57 --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.h @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTP_RTCP_IMPL2_H_ +#define MODULES_RTP_RTCP_SOURCE_RTP_RTCP_IMPL2_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/rtp_headers.h" +#include "api/video/video_bitrate_allocation.h" +#include "modules/include/module_fec_types.h" +#include "modules/remote_bitrate_estimator/include/remote_bitrate_estimator.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" // RTCPPacketType +#include "modules/rtp_rtcp/source/rtcp_packet/tmmb_item.h" +#include "modules/rtp_rtcp/source/rtcp_receiver.h" +#include "modules/rtp_rtcp/source/rtcp_sender.h" +#include "modules/rtp_rtcp/source/rtp_packet_history.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" +#include "modules/rtp_rtcp/source/rtp_sender.h" +#include "modules/rtp_rtcp/source/rtp_sender_egress.h" +#include "rtc_base/critical_section.h" +#include "rtc_base/gtest_prod_util.h" +#include "rtc_base/synchronization/sequence_checker.h" + +namespace webrtc { + +class Clock; +struct PacedPacketInfo; +struct RTPVideoHeader; + +class ModuleRtpRtcpImpl2 final : public RtpRtcpInterface, + public Module, + public RTCPReceiver::ModuleRtpRtcp { + public: + explicit ModuleRtpRtcpImpl2( + const RtpRtcpInterface::Configuration& configuration); + ~ModuleRtpRtcpImpl2() override; + + // This method is provided to easy with migrating away from the + // RtpRtcp::Create factory method. Since this is an internal implementation + // detail though, creating an instance of ModuleRtpRtcpImpl2 directly should + // be fine. + static std::unique_ptr<ModuleRtpRtcpImpl2> Create( + const Configuration& configuration); + + // Returns the number of milliseconds until the module want a worker thread to + // call Process. + int64_t TimeUntilNextProcess() override; + + // Process any pending tasks such as timeouts. + void Process() override; + + // Receiver part. + + // Called when we receive an RTCP packet. + void IncomingRtcpPacket(const uint8_t* incoming_packet, + size_t incoming_packet_length) override; + + void SetRemoteSSRC(uint32_t ssrc) override; + + // Sender part. + void RegisterSendPayloadFrequency(int payload_type, + int payload_frequency) override; + + int32_t DeRegisterSendPayload(int8_t payload_type) override; + + void SetExtmapAllowMixed(bool extmap_allow_mixed) override; + + void RegisterRtpHeaderExtension(absl::string_view uri, int id) override; + int32_t DeregisterSendRtpHeaderExtension(RTPExtensionType type) override; + void DeregisterSendRtpHeaderExtension(absl::string_view uri) override; + + bool SupportsPadding() const override; + bool SupportsRtxPayloadPadding() const override; + + // Get start timestamp. + uint32_t StartTimestamp() const override; + + // Configure start timestamp, default is a random number. + void SetStartTimestamp(uint32_t timestamp) override; + + uint16_t SequenceNumber() const override; + + // Set SequenceNumber, default is a random number. + void SetSequenceNumber(uint16_t seq) override; + + void SetRtpState(const RtpState& rtp_state) override; + void SetRtxState(const RtpState& rtp_state) override; + RtpState GetRtpState() const override; + RtpState GetRtxState() const override; + + uint32_t SSRC() const override { return rtcp_sender_.SSRC(); } + + void SetRid(const std::string& rid) override; + + void SetMid(const std::string& mid) override; + + void SetCsrcs(const std::vector<uint32_t>& csrcs) override; + + RTCPSender::FeedbackState GetFeedbackState(); + + void SetRtxSendStatus(int mode) override; + int RtxSendStatus() const override; + absl::optional<uint32_t> RtxSsrc() const override; + + void SetRtxSendPayloadType(int payload_type, + int associated_payload_type) override; + + absl::optional<uint32_t> FlexfecSsrc() const override; + + // Sends kRtcpByeCode when going from true to false. + int32_t SetSendingStatus(bool sending) override; + + bool Sending() const override; + + // Drops or relays media packets. + void SetSendingMediaStatus(bool sending) override; + + bool SendingMedia() const override; + + bool IsAudioConfigured() const override; + + void SetAsPartOfAllocation(bool part_of_allocation) override; + + bool OnSendingRtpFrame(uint32_t timestamp, + int64_t capture_time_ms, + int payload_type, + bool force_sender_report) override; + + bool TrySendPacket(RtpPacketToSend* packet, + const PacedPacketInfo& pacing_info) override; + + void OnPacketsAcknowledged( + rtc::ArrayView<const uint16_t> sequence_numbers) override; + + std::vector<std::unique_ptr<RtpPacketToSend>> GeneratePadding( + size_t target_size_bytes) override; + + std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( + rtc::ArrayView<const uint16_t> sequence_numbers) const override; + + size_t ExpectedPerPacketOverhead() const override; + + // RTCP part. + + // Get RTCP status. + RtcpMode RTCP() const override; + + // Configure RTCP status i.e on/off. + void SetRTCPStatus(RtcpMode method) override; + + // Set RTCP CName. + int32_t SetCNAME(const char* c_name) override; + + // Get remote NTP. + int32_t RemoteNTP(uint32_t* received_ntp_secs, + uint32_t* received_ntp_frac, + uint32_t* rtcp_arrival_time_secs, + uint32_t* rtcp_arrival_time_frac, + uint32_t* rtcp_timestamp) const override; + + // Get RoundTripTime. + int32_t RTT(uint32_t remote_ssrc, + int64_t* rtt, + int64_t* avg_rtt, + int64_t* min_rtt, + int64_t* max_rtt) const override; + + int64_t ExpectedRetransmissionTimeMs() const override; + + // Force a send of an RTCP packet. + // Normal SR and RR are triggered via the process function. + int32_t SendRTCP(RTCPPacketType rtcpPacketType) override; + + void GetSendStreamDataCounters( + StreamDataCounters* rtp_counters, + StreamDataCounters* rtx_counters) const override; + + // Get received RTCP report, report block. + int32_t RemoteRTCPStat( + std::vector<RTCPReportBlock>* receive_blocks) const override; + // A snapshot of the most recent Report Block with additional data of + // interest to statistics. Used to implement RTCRemoteInboundRtpStreamStats. + // Within this list, the ReportBlockData::RTCPReportBlock::source_ssrc(), + // which is the SSRC of the corresponding outbound RTP stream, is unique. + std::vector<ReportBlockData> GetLatestReportBlockData() const override; + + // (REMB) Receiver Estimated Max Bitrate. + void SetRemb(int64_t bitrate_bps, std::vector<uint32_t> ssrcs) override; + void UnsetRemb() override; + + void SetTmmbn(std::vector<rtcp::TmmbItem> bounding_set) override; + + size_t MaxRtpPacketSize() const override; + + void SetMaxRtpPacketSize(size_t max_packet_size) override; + + // (NACK) Negative acknowledgment part. + + // Send a Negative acknowledgment packet. + // TODO(philipel): Deprecate SendNACK and use SendNack instead. + int32_t SendNACK(const uint16_t* nack_list, uint16_t size) override; + + void SendNack(const std::vector<uint16_t>& sequence_numbers) override; + + // Store the sent packets, needed to answer to a negative acknowledgment + // requests. + void SetStorePacketsStatus(bool enable, uint16_t number_to_store) override; + + bool StorePackets() const override; + + void SendCombinedRtcpPacket( + std::vector<std::unique_ptr<rtcp::RtcpPacket>> rtcp_packets) override; + + // (XR) Receiver reference time report. + void SetRtcpXrRrtrStatus(bool enable) override; + + bool RtcpXrRrtrStatus() const override; + + // Video part. + int32_t SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) override; + + bool LastReceivedNTP(uint32_t* NTPsecs, + uint32_t* NTPfrac, + uint32_t* remote_sr) const; + + void BitrateSent(uint32_t* total_rate, + uint32_t* video_rate, + uint32_t* fec_rate, + uint32_t* nackRate) const override; + + RtpSendRates GetSendRates() const override; + + void OnReceivedNack( + const std::vector<uint16_t>& nack_sequence_numbers) override; + void OnReceivedRtcpReportBlocks( + const ReportBlockList& report_blocks) override; + void OnRequestSendReport() override; + + void SetVideoBitrateAllocation( + const VideoBitrateAllocation& bitrate) override; + + RTPSender* RtpSender() override; + const RTPSender* RtpSender() const override; + + private: + FRIEND_TEST_ALL_PREFIXES(RtpRtcpImpl2Test, Rtt); + FRIEND_TEST_ALL_PREFIXES(RtpRtcpImpl2Test, RttForReceiverOnly); + + struct RtpSenderContext { + explicit RtpSenderContext(const RtpRtcpInterface::Configuration& config); + // Storage of packets, for retransmissions and padding, if applicable. + RtpPacketHistory packet_history; + // Handles final time timestamping/stats/etc and handover to Transport. + RtpSenderEgress packet_sender; + // If no paced sender configured, this class will be used to pass packets + // from |packet_generator_| to |packet_sender_|. + RtpSenderEgress::NonPacedPacketSender non_paced_sender; + // Handles creation of RTP packets to be sent. + RTPSender packet_generator; + }; + + void set_rtt_ms(int64_t rtt_ms); + int64_t rtt_ms() const; + + bool TimeToSendFullNackList(int64_t now) const; + + SequenceChecker construction_thread_checker_; + SequenceChecker process_thread_checker_; + + std::unique_ptr<RtpSenderContext> rtp_sender_; + + RTCPSender rtcp_sender_; + RTCPReceiver rtcp_receiver_; + + Clock* const clock_; + + int64_t last_bitrate_process_time_; + int64_t last_rtt_process_time_; + int64_t next_process_time_; + uint16_t packet_overhead_; + + // Send side + int64_t nack_last_time_sent_full_ms_; + uint16_t nack_last_seq_number_sent_; + + RemoteBitrateEstimator* const remote_bitrate_; + + RtcpRttStats* const rtt_stats_; + + // The processed RTT from RtcpRttStats. + rtc::CriticalSection critical_section_rtt_; + int64_t rtt_ms_; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_RTP_RTCP_IMPL2_H_ diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2_unittest.cc new file mode 100644 index 00000000000..5861ae99caa --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2_unittest.cc @@ -0,0 +1,630 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" + +#include <map> +#include <memory> +#include <set> + +#include "api/transport/field_trial_based_config.h" +#include "api/video_codecs/video_codec.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtcp_packet.h" +#include "modules/rtp_rtcp/source/rtcp_packet/nack.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_sender_video.h" +#include "rtc_base/rate_limiter.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/rtcp_packet_parser.h" +#include "test/rtp_header_parser.h" + +using ::testing::ElementsAre; + +namespace webrtc { +namespace { +const uint32_t kSenderSsrc = 0x12345; +const uint32_t kReceiverSsrc = 0x23456; +const int64_t kOneWayNetworkDelayMs = 100; +const uint8_t kBaseLayerTid = 0; +const uint8_t kHigherLayerTid = 1; +const uint16_t kSequenceNumber = 100; + +class RtcpRttStatsTestImpl : public RtcpRttStats { + public: + RtcpRttStatsTestImpl() : rtt_ms_(0) {} + ~RtcpRttStatsTestImpl() override = default; + + void OnRttUpdate(int64_t rtt_ms) override { rtt_ms_ = rtt_ms; } + int64_t LastProcessedRtt() const override { return rtt_ms_; } + int64_t rtt_ms_; +}; + +class SendTransport : public Transport { + public: + SendTransport() + : receiver_(nullptr), + clock_(nullptr), + delay_ms_(0), + rtp_packets_sent_(0), + rtcp_packets_sent_(0) {} + + void SetRtpRtcpModule(ModuleRtpRtcpImpl2* receiver) { receiver_ = receiver; } + void SimulateNetworkDelay(int64_t delay_ms, SimulatedClock* clock) { + clock_ = clock; + delay_ms_ = delay_ms; + } + bool SendRtp(const uint8_t* data, + size_t len, + const PacketOptions& options) override { + RTPHeader header; + std::unique_ptr<RtpHeaderParser> parser(RtpHeaderParser::CreateForTest()); + EXPECT_TRUE(parser->Parse(static_cast<const uint8_t*>(data), len, &header)); + ++rtp_packets_sent_; + last_rtp_header_ = header; + return true; + } + bool SendRtcp(const uint8_t* data, size_t len) override { + test::RtcpPacketParser parser; + parser.Parse(data, len); + last_nack_list_ = parser.nack()->packet_ids(); + + if (clock_) { + clock_->AdvanceTimeMilliseconds(delay_ms_); + } + EXPECT_TRUE(receiver_); + receiver_->IncomingRtcpPacket(data, len); + ++rtcp_packets_sent_; + return true; + } + size_t NumRtcpSent() { return rtcp_packets_sent_; } + ModuleRtpRtcpImpl2* receiver_; + SimulatedClock* clock_; + int64_t delay_ms_; + int rtp_packets_sent_; + size_t rtcp_packets_sent_; + RTPHeader last_rtp_header_; + std::vector<uint16_t> last_nack_list_; +}; + +class RtpRtcpModule : public RtcpPacketTypeCounterObserver { + public: + RtpRtcpModule(SimulatedClock* clock, bool is_sender) + : is_sender_(is_sender), + receive_statistics_(ReceiveStatistics::Create(clock)), + clock_(clock) { + CreateModuleImpl(); + transport_.SimulateNetworkDelay(kOneWayNetworkDelayMs, clock); + } + + const bool is_sender_; + RtcpPacketTypeCounter packets_sent_; + RtcpPacketTypeCounter packets_received_; + std::unique_ptr<ReceiveStatistics> receive_statistics_; + SendTransport transport_; + RtcpRttStatsTestImpl rtt_stats_; + std::unique_ptr<ModuleRtpRtcpImpl2> impl_; + int rtcp_report_interval_ms_ = 0; + + void RtcpPacketTypesCounterUpdated( + uint32_t ssrc, + const RtcpPacketTypeCounter& packet_counter) override { + counter_map_[ssrc] = packet_counter; + } + + RtcpPacketTypeCounter RtcpSent() { + // RTCP counters for remote SSRC. + return counter_map_[is_sender_ ? kReceiverSsrc : kSenderSsrc]; + } + + RtcpPacketTypeCounter RtcpReceived() { + // Received RTCP stats for (own) local SSRC. + return counter_map_[impl_->SSRC()]; + } + int RtpSent() { return transport_.rtp_packets_sent_; } + uint16_t LastRtpSequenceNumber() { + return transport_.last_rtp_header_.sequenceNumber; + } + std::vector<uint16_t> LastNackListSent() { + return transport_.last_nack_list_; + } + void SetRtcpReportIntervalAndReset(int rtcp_report_interval_ms) { + rtcp_report_interval_ms_ = rtcp_report_interval_ms; + CreateModuleImpl(); + } + + private: + void CreateModuleImpl() { + RtpRtcpInterface::Configuration config; + config.audio = false; + config.clock = clock_; + config.outgoing_transport = &transport_; + config.receive_statistics = receive_statistics_.get(); + config.rtcp_packet_type_counter_observer = this; + config.rtt_stats = &rtt_stats_; + config.rtcp_report_interval_ms = rtcp_report_interval_ms_; + config.local_media_ssrc = is_sender_ ? kSenderSsrc : kReceiverSsrc; + config.need_rtp_packet_infos = true; + + impl_.reset(new ModuleRtpRtcpImpl2(config)); + impl_->SetRemoteSSRC(is_sender_ ? kReceiverSsrc : kSenderSsrc); + impl_->SetRTCPStatus(RtcpMode::kCompound); + } + + SimulatedClock* const clock_; + std::map<uint32_t, RtcpPacketTypeCounter> counter_map_; +}; +} // namespace + +class RtpRtcpImpl2Test : public ::testing::Test { + protected: + RtpRtcpImpl2Test() + : clock_(133590000000000), + sender_(&clock_, /*is_sender=*/true), + receiver_(&clock_, /*is_sender=*/false) {} + + void SetUp() override { + // Send module. + EXPECT_EQ(0, sender_.impl_->SetSendingStatus(true)); + sender_.impl_->SetSendingMediaStatus(true); + sender_.impl_->SetSequenceNumber(kSequenceNumber); + sender_.impl_->SetStorePacketsStatus(true, 100); + + FieldTrialBasedConfig field_trials; + RTPSenderVideo::Config video_config; + video_config.clock = &clock_; + video_config.rtp_sender = sender_.impl_->RtpSender(); + video_config.field_trials = &field_trials; + sender_video_ = std::make_unique<RTPSenderVideo>(video_config); + + memset(&codec_, 0, sizeof(VideoCodec)); + codec_.plType = 100; + codec_.width = 320; + codec_.height = 180; + + // Receive module. + EXPECT_EQ(0, receiver_.impl_->SetSendingStatus(false)); + receiver_.impl_->SetSendingMediaStatus(false); + // Transport settings. + sender_.transport_.SetRtpRtcpModule(receiver_.impl_.get()); + receiver_.transport_.SetRtpRtcpModule(sender_.impl_.get()); + } + + SimulatedClock clock_; + RtpRtcpModule sender_; + std::unique_ptr<RTPSenderVideo> sender_video_; + RtpRtcpModule receiver_; + VideoCodec codec_; + + void SendFrame(const RtpRtcpModule* module, + RTPSenderVideo* sender, + uint8_t tid) { + RTPVideoHeaderVP8 vp8_header = {}; + vp8_header.temporalIdx = tid; + RTPVideoHeader rtp_video_header; + rtp_video_header.frame_type = VideoFrameType::kVideoFrameKey; + rtp_video_header.width = codec_.width; + rtp_video_header.height = codec_.height; + rtp_video_header.rotation = kVideoRotation_0; + rtp_video_header.content_type = VideoContentType::UNSPECIFIED; + rtp_video_header.playout_delay = {-1, -1}; + rtp_video_header.is_first_packet_in_frame = true; + rtp_video_header.simulcastIdx = 0; + rtp_video_header.codec = kVideoCodecVP8; + rtp_video_header.video_type_header = vp8_header; + rtp_video_header.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; + + const uint8_t payload[100] = {0}; + EXPECT_TRUE(module->impl_->OnSendingRtpFrame(0, 0, codec_.plType, true)); + EXPECT_TRUE(sender->SendVideo(codec_.plType, VideoCodecType::kVideoCodecVP8, + 0, 0, payload, nullptr, rtp_video_header, 0)); + } + + void IncomingRtcpNack(const RtpRtcpModule* module, uint16_t sequence_number) { + bool sender = module->impl_->SSRC() == kSenderSsrc; + rtcp::Nack nack; + uint16_t list[1]; + list[0] = sequence_number; + const uint16_t kListLength = sizeof(list) / sizeof(list[0]); + nack.SetSenderSsrc(sender ? kReceiverSsrc : kSenderSsrc); + nack.SetMediaSsrc(sender ? kSenderSsrc : kReceiverSsrc); + nack.SetPacketIds(list, kListLength); + rtc::Buffer packet = nack.Build(); + module->impl_->IncomingRtcpPacket(packet.data(), packet.size()); + } +}; + +TEST_F(RtpRtcpImpl2Test, RetransmitsAllLayers) { + // Send frames. + EXPECT_EQ(0, sender_.RtpSent()); + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); // kSequenceNumber + SendFrame(&sender_, sender_video_.get(), + kHigherLayerTid); // kSequenceNumber + 1 + SendFrame(&sender_, sender_video_.get(), + kNoTemporalIdx); // kSequenceNumber + 2 + EXPECT_EQ(3, sender_.RtpSent()); + EXPECT_EQ(kSequenceNumber + 2, sender_.LastRtpSequenceNumber()); + + // Min required delay until retransmit = 5 + RTT ms (RTT = 0). + clock_.AdvanceTimeMilliseconds(5); + + // Frame with kBaseLayerTid re-sent. + IncomingRtcpNack(&sender_, kSequenceNumber); + EXPECT_EQ(4, sender_.RtpSent()); + EXPECT_EQ(kSequenceNumber, sender_.LastRtpSequenceNumber()); + // Frame with kHigherLayerTid re-sent. + IncomingRtcpNack(&sender_, kSequenceNumber + 1); + EXPECT_EQ(5, sender_.RtpSent()); + EXPECT_EQ(kSequenceNumber + 1, sender_.LastRtpSequenceNumber()); + // Frame with kNoTemporalIdx re-sent. + IncomingRtcpNack(&sender_, kSequenceNumber + 2); + EXPECT_EQ(6, sender_.RtpSent()); + EXPECT_EQ(kSequenceNumber + 2, sender_.LastRtpSequenceNumber()); +} + +TEST_F(RtpRtcpImpl2Test, Rtt) { + RtpPacketReceived packet; + packet.SetTimestamp(1); + packet.SetSequenceNumber(123); + packet.SetSsrc(kSenderSsrc); + packet.AllocatePayload(100 - 12); + receiver_.receive_statistics_->OnRtpPacket(packet); + + // Send Frame before sending an SR. + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + // Sender module should send an SR. + EXPECT_EQ(0, sender_.impl_->SendRTCP(kRtcpReport)); + + // Receiver module should send a RR with a response to the last received SR. + clock_.AdvanceTimeMilliseconds(1000); + EXPECT_EQ(0, receiver_.impl_->SendRTCP(kRtcpReport)); + + // Verify RTT. + int64_t rtt; + int64_t avg_rtt; + int64_t min_rtt; + int64_t max_rtt; + EXPECT_EQ( + 0, sender_.impl_->RTT(kReceiverSsrc, &rtt, &avg_rtt, &min_rtt, &max_rtt)); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, rtt, 1); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, avg_rtt, 1); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, min_rtt, 1); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, max_rtt, 1); + + // No RTT from other ssrc. + EXPECT_EQ(-1, sender_.impl_->RTT(kReceiverSsrc + 1, &rtt, &avg_rtt, &min_rtt, + &max_rtt)); + + // Verify RTT from rtt_stats config. + EXPECT_EQ(0, sender_.rtt_stats_.LastProcessedRtt()); + EXPECT_EQ(0, sender_.impl_->rtt_ms()); + sender_.impl_->Process(); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, sender_.rtt_stats_.LastProcessedRtt(), + 1); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, sender_.impl_->rtt_ms(), 1); +} + +TEST_F(RtpRtcpImpl2Test, SetRtcpXrRrtrStatus) { + EXPECT_FALSE(receiver_.impl_->RtcpXrRrtrStatus()); + receiver_.impl_->SetRtcpXrRrtrStatus(true); + EXPECT_TRUE(receiver_.impl_->RtcpXrRrtrStatus()); +} + +TEST_F(RtpRtcpImpl2Test, RttForReceiverOnly) { + receiver_.impl_->SetRtcpXrRrtrStatus(true); + + // Receiver module should send a Receiver time reference report (RTRR). + EXPECT_EQ(0, receiver_.impl_->SendRTCP(kRtcpReport)); + + // Sender module should send a response to the last received RTRR (DLRR). + clock_.AdvanceTimeMilliseconds(1000); + // Send Frame before sending a SR. + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + EXPECT_EQ(0, sender_.impl_->SendRTCP(kRtcpReport)); + + // Verify RTT. + EXPECT_EQ(0, receiver_.rtt_stats_.LastProcessedRtt()); + EXPECT_EQ(0, receiver_.impl_->rtt_ms()); + receiver_.impl_->Process(); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, + receiver_.rtt_stats_.LastProcessedRtt(), 1); + EXPECT_NEAR(2 * kOneWayNetworkDelayMs, receiver_.impl_->rtt_ms(), 1); +} + +TEST_F(RtpRtcpImpl2Test, NoSrBeforeMedia) { + // Ignore fake transport delays in this test. + sender_.transport_.SimulateNetworkDelay(0, &clock_); + receiver_.transport_.SimulateNetworkDelay(0, &clock_); + + sender_.impl_->Process(); + EXPECT_EQ(-1, sender_.RtcpSent().first_packet_time_ms); + + // Verify no SR is sent before media has been sent, RR should still be sent + // from the receiving module though. + clock_.AdvanceTimeMilliseconds(2000); + int64_t current_time = clock_.TimeInMilliseconds(); + sender_.impl_->Process(); + receiver_.impl_->Process(); + EXPECT_EQ(-1, sender_.RtcpSent().first_packet_time_ms); + EXPECT_EQ(receiver_.RtcpSent().first_packet_time_ms, current_time); + + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + EXPECT_EQ(sender_.RtcpSent().first_packet_time_ms, current_time); +} + +TEST_F(RtpRtcpImpl2Test, RtcpPacketTypeCounter_Nack) { + EXPECT_EQ(-1, receiver_.RtcpSent().first_packet_time_ms); + EXPECT_EQ(-1, sender_.RtcpReceived().first_packet_time_ms); + EXPECT_EQ(0U, sender_.RtcpReceived().nack_packets); + EXPECT_EQ(0U, receiver_.RtcpSent().nack_packets); + + // Receive module sends a NACK. + const uint16_t kNackLength = 1; + uint16_t nack_list[kNackLength] = {123}; + EXPECT_EQ(0, receiver_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, receiver_.RtcpSent().nack_packets); + EXPECT_GT(receiver_.RtcpSent().first_packet_time_ms, -1); + + // Send module receives the NACK. + EXPECT_EQ(1U, sender_.RtcpReceived().nack_packets); + EXPECT_GT(sender_.RtcpReceived().first_packet_time_ms, -1); +} + +TEST_F(RtpRtcpImpl2Test, AddStreamDataCounters) { + StreamDataCounters rtp; + const int64_t kStartTimeMs = 1; + rtp.first_packet_time_ms = kStartTimeMs; + rtp.transmitted.packets = 1; + rtp.transmitted.payload_bytes = 1; + rtp.transmitted.header_bytes = 2; + rtp.transmitted.padding_bytes = 3; + EXPECT_EQ(rtp.transmitted.TotalBytes(), rtp.transmitted.payload_bytes + + rtp.transmitted.header_bytes + + rtp.transmitted.padding_bytes); + + StreamDataCounters rtp2; + rtp2.first_packet_time_ms = -1; + rtp2.transmitted.packets = 10; + rtp2.transmitted.payload_bytes = 10; + rtp2.retransmitted.header_bytes = 4; + rtp2.retransmitted.payload_bytes = 5; + rtp2.retransmitted.padding_bytes = 6; + rtp2.retransmitted.packets = 7; + rtp2.fec.packets = 8; + + StreamDataCounters sum = rtp; + sum.Add(rtp2); + EXPECT_EQ(kStartTimeMs, sum.first_packet_time_ms); + EXPECT_EQ(11U, sum.transmitted.packets); + EXPECT_EQ(11U, sum.transmitted.payload_bytes); + EXPECT_EQ(2U, sum.transmitted.header_bytes); + EXPECT_EQ(3U, sum.transmitted.padding_bytes); + EXPECT_EQ(4U, sum.retransmitted.header_bytes); + EXPECT_EQ(5U, sum.retransmitted.payload_bytes); + EXPECT_EQ(6U, sum.retransmitted.padding_bytes); + EXPECT_EQ(7U, sum.retransmitted.packets); + EXPECT_EQ(8U, sum.fec.packets); + EXPECT_EQ(sum.transmitted.TotalBytes(), + rtp.transmitted.TotalBytes() + rtp2.transmitted.TotalBytes()); + + StreamDataCounters rtp3; + rtp3.first_packet_time_ms = kStartTimeMs + 10; + sum.Add(rtp3); + EXPECT_EQ(kStartTimeMs, sum.first_packet_time_ms); // Holds oldest time. +} + +TEST_F(RtpRtcpImpl2Test, SendsInitialNackList) { + // Send module sends a NACK. + const uint16_t kNackLength = 1; + uint16_t nack_list[kNackLength] = {123}; + EXPECT_EQ(0U, sender_.RtcpSent().nack_packets); + // Send Frame before sending a compound RTCP that starts with SR. + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); + EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123)); +} + +TEST_F(RtpRtcpImpl2Test, SendsExtendedNackList) { + // Send module sends a NACK. + const uint16_t kNackLength = 1; + uint16_t nack_list[kNackLength] = {123}; + EXPECT_EQ(0U, sender_.RtcpSent().nack_packets); + // Send Frame before sending a compound RTCP that starts with SR. + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); + EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123)); + + // Same list not re-send. + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); + EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123)); + + // Only extended list sent. + const uint16_t kNackExtLength = 2; + uint16_t nack_list_ext[kNackExtLength] = {123, 124}; + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list_ext, kNackExtLength)); + EXPECT_EQ(2U, sender_.RtcpSent().nack_packets); + EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(124)); +} + +TEST_F(RtpRtcpImpl2Test, ReSendsNackListAfterRttMs) { + sender_.transport_.SimulateNetworkDelay(0, &clock_); + // Send module sends a NACK. + const uint16_t kNackLength = 2; + uint16_t nack_list[kNackLength] = {123, 125}; + EXPECT_EQ(0U, sender_.RtcpSent().nack_packets); + // Send Frame before sending a compound RTCP that starts with SR. + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); + EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123, 125)); + + // Same list not re-send, rtt interval has not passed. + const int kStartupRttMs = 100; + clock_.AdvanceTimeMilliseconds(kStartupRttMs); + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, sender_.RtcpSent().nack_packets); + + // Rtt interval passed, full list sent. + clock_.AdvanceTimeMilliseconds(1); + EXPECT_EQ(0, sender_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(2U, sender_.RtcpSent().nack_packets); + EXPECT_THAT(sender_.LastNackListSent(), ElementsAre(123, 125)); +} + +TEST_F(RtpRtcpImpl2Test, UniqueNackRequests) { + receiver_.transport_.SimulateNetworkDelay(0, &clock_); + EXPECT_EQ(0U, receiver_.RtcpSent().nack_packets); + EXPECT_EQ(0U, receiver_.RtcpSent().nack_requests); + EXPECT_EQ(0U, receiver_.RtcpSent().unique_nack_requests); + EXPECT_EQ(0, receiver_.RtcpSent().UniqueNackRequestsInPercent()); + + // Receive module sends NACK request. + const uint16_t kNackLength = 4; + uint16_t nack_list[kNackLength] = {10, 11, 13, 18}; + EXPECT_EQ(0, receiver_.impl_->SendNACK(nack_list, kNackLength)); + EXPECT_EQ(1U, receiver_.RtcpSent().nack_packets); + EXPECT_EQ(4U, receiver_.RtcpSent().nack_requests); + EXPECT_EQ(4U, receiver_.RtcpSent().unique_nack_requests); + EXPECT_THAT(receiver_.LastNackListSent(), ElementsAre(10, 11, 13, 18)); + + // Send module receives the request. + EXPECT_EQ(1U, sender_.RtcpReceived().nack_packets); + EXPECT_EQ(4U, sender_.RtcpReceived().nack_requests); + EXPECT_EQ(4U, sender_.RtcpReceived().unique_nack_requests); + EXPECT_EQ(100, sender_.RtcpReceived().UniqueNackRequestsInPercent()); + + // Receive module sends new request with duplicated packets. + const int kStartupRttMs = 100; + clock_.AdvanceTimeMilliseconds(kStartupRttMs + 1); + const uint16_t kNackLength2 = 4; + uint16_t nack_list2[kNackLength2] = {11, 18, 20, 21}; + EXPECT_EQ(0, receiver_.impl_->SendNACK(nack_list2, kNackLength2)); + EXPECT_EQ(2U, receiver_.RtcpSent().nack_packets); + EXPECT_EQ(8U, receiver_.RtcpSent().nack_requests); + EXPECT_EQ(6U, receiver_.RtcpSent().unique_nack_requests); + EXPECT_THAT(receiver_.LastNackListSent(), ElementsAre(11, 18, 20, 21)); + + // Send module receives the request. + EXPECT_EQ(2U, sender_.RtcpReceived().nack_packets); + EXPECT_EQ(8U, sender_.RtcpReceived().nack_requests); + EXPECT_EQ(6U, sender_.RtcpReceived().unique_nack_requests); + EXPECT_EQ(75, sender_.RtcpReceived().UniqueNackRequestsInPercent()); +} + +TEST_F(RtpRtcpImpl2Test, ConfigurableRtcpReportInterval) { + const int kVideoReportInterval = 3000; + + // Recreate sender impl with new configuration, and redo setup. + sender_.SetRtcpReportIntervalAndReset(kVideoReportInterval); + SetUp(); + + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + + // Initial state + sender_.impl_->Process(); + EXPECT_EQ(sender_.RtcpSent().first_packet_time_ms, -1); + EXPECT_EQ(0u, sender_.transport_.NumRtcpSent()); + + // Move ahead to the last ms before a rtcp is expected, no action. + clock_.AdvanceTimeMilliseconds(kVideoReportInterval / 2 - 1); + sender_.impl_->Process(); + EXPECT_EQ(sender_.RtcpSent().first_packet_time_ms, -1); + EXPECT_EQ(sender_.transport_.NumRtcpSent(), 0u); + + // Move ahead to the first rtcp. Send RTCP. + clock_.AdvanceTimeMilliseconds(1); + sender_.impl_->Process(); + EXPECT_GT(sender_.RtcpSent().first_packet_time_ms, -1); + EXPECT_EQ(sender_.transport_.NumRtcpSent(), 1u); + + SendFrame(&sender_, sender_video_.get(), kBaseLayerTid); + + // Move ahead to the last possible second before second rtcp is expected. + clock_.AdvanceTimeMilliseconds(kVideoReportInterval * 1 / 2 - 1); + sender_.impl_->Process(); + EXPECT_EQ(sender_.transport_.NumRtcpSent(), 1u); + + // Move ahead into the range of second rtcp, the second rtcp may be sent. + clock_.AdvanceTimeMilliseconds(1); + sender_.impl_->Process(); + EXPECT_GE(sender_.transport_.NumRtcpSent(), 1u); + + clock_.AdvanceTimeMilliseconds(kVideoReportInterval / 2); + sender_.impl_->Process(); + EXPECT_GE(sender_.transport_.NumRtcpSent(), 1u); + + // Move out the range of second rtcp, the second rtcp must have been sent. + clock_.AdvanceTimeMilliseconds(kVideoReportInterval / 2); + sender_.impl_->Process(); + EXPECT_EQ(sender_.transport_.NumRtcpSent(), 2u); +} + +TEST_F(RtpRtcpImpl2Test, StoresPacketInfoForSentPackets) { + const uint32_t kStartTimestamp = 1u; + SetUp(); + sender_.impl_->SetStartTimestamp(kStartTimestamp); + + PacedPacketInfo pacing_info; + RtpPacketToSend packet(nullptr); + packet.set_packet_type(RtpPacketToSend::Type::kVideo); + packet.SetSsrc(kSenderSsrc); + + // Single-packet frame. + packet.SetTimestamp(1); + packet.SetSequenceNumber(1); + packet.set_first_packet_of_frame(true); + packet.SetMarker(true); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + std::vector<RtpSequenceNumberMap::Info> seqno_info = + sender_.impl_->GetSentRtpPacketInfos(std::vector<uint16_t>{1}); + + EXPECT_THAT(seqno_info, ElementsAre(RtpSequenceNumberMap::Info( + /*timestamp=*/1 - kStartTimestamp, + /*is_first=*/1, + /*is_last=*/1))); + + // Three-packet frame. + packet.SetTimestamp(2); + packet.SetSequenceNumber(2); + packet.set_first_packet_of_frame(true); + packet.SetMarker(false); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + packet.SetSequenceNumber(3); + packet.set_first_packet_of_frame(false); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + packet.SetSequenceNumber(4); + packet.SetMarker(true); + sender_.impl_->TrySendPacket(&packet, pacing_info); + + seqno_info = + sender_.impl_->GetSentRtpPacketInfos(std::vector<uint16_t>{2, 3, 4}); + + EXPECT_THAT(seqno_info, ElementsAre(RtpSequenceNumberMap::Info( + /*timestamp=*/2 - kStartTimestamp, + /*is_first=*/1, + /*is_last=*/0), + RtpSequenceNumberMap::Info( + /*timestamp=*/2 - kStartTimestamp, + /*is_first=*/0, + /*is_last=*/0), + RtpSequenceNumberMap::Info( + /*timestamp=*/2 - kStartTimestamp, + /*is_first=*/0, + /*is_last=*/1))); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc index e2595664f6e..dd7b512ff21 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_impl_unittest.cc @@ -143,7 +143,7 @@ class RtpRtcpModule : public RtcpPacketTypeCounterObserver { private: void CreateModuleImpl() { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.audio = false; config.clock = clock_; config.outgoing_transport = &transport_; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_interface.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_interface.h new file mode 100644 index 00000000000..440837fc5df --- /dev/null +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_rtcp_interface.h @@ -0,0 +1,424 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTP_RTCP_INTERFACE_H_ +#define MODULES_RTP_RTCP_SOURCE_RTP_RTCP_INTERFACE_H_ + +#include <memory> +#include <string> +#include <vector> + +#include "absl/types/optional.h" +#include "api/frame_transformer_interface.h" +#include "api/scoped_refptr.h" +#include "api/transport/webrtc_key_value_config.h" +#include "api/video/video_bitrate_allocation.h" +#include "modules/rtp_rtcp/include/receive_statistics.h" +#include "modules/rtp_rtcp/include/report_block_data.h" +#include "modules/rtp_rtcp/include/rtp_packet_sender.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" +#include "modules/rtp_rtcp/source/video_fec_generator.h" +#include "rtc_base/constructor_magic.h" + +namespace webrtc { + +// Forward declarations. +class FrameEncryptorInterface; +class RateLimiter; +class RemoteBitrateEstimator; +class RtcEventLog; +class RTPSender; +class Transport; +class VideoBitrateAllocationObserver; + +class RtpRtcpInterface : public RtcpFeedbackSenderInterface { + public: + struct Configuration { + Configuration() = default; + Configuration(Configuration&& rhs) = default; + + // True for a audio version of the RTP/RTCP module object false will create + // a video version. + bool audio = false; + bool receiver_only = false; + + // The clock to use to read time. If nullptr then system clock will be used. + Clock* clock = nullptr; + + ReceiveStatisticsProvider* receive_statistics = nullptr; + + // Transport object that will be called when packets are ready to be sent + // out on the network. + Transport* outgoing_transport = nullptr; + + // Called when the receiver requests an intra frame. + RtcpIntraFrameObserver* intra_frame_callback = nullptr; + + // Called when the receiver sends a loss notification. + RtcpLossNotificationObserver* rtcp_loss_notification_observer = nullptr; + + // Called when we receive a changed estimate from the receiver of out + // stream. + RtcpBandwidthObserver* bandwidth_callback = nullptr; + + NetworkStateEstimateObserver* network_state_estimate_observer = nullptr; + TransportFeedbackObserver* transport_feedback_callback = nullptr; + VideoBitrateAllocationObserver* bitrate_allocation_observer = nullptr; + RtcpRttStats* rtt_stats = nullptr; + RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer = nullptr; + // Called on receipt of RTCP report block from remote side. + // TODO(bugs.webrtc.org/10678): Remove RtcpStatisticsCallback in + // favor of ReportBlockDataObserver. + // TODO(bugs.webrtc.org/10679): Consider whether we want to use + // only getters or only callbacks. If we decide on getters, the + // ReportBlockDataObserver should also be removed in favor of + // GetLatestReportBlockData(). + RtcpStatisticsCallback* rtcp_statistics_callback = nullptr; + RtcpCnameCallback* rtcp_cname_callback = nullptr; + ReportBlockDataObserver* report_block_data_observer = nullptr; + + // Estimates the bandwidth available for a set of streams from the same + // client. + RemoteBitrateEstimator* remote_bitrate_estimator = nullptr; + + // Spread any bursts of packets into smaller bursts to minimize packet loss. + RtpPacketSender* paced_sender = nullptr; + + // Generates FEC packets. + // TODO(sprang): Wire up to RtpSenderEgress. + VideoFecGenerator* fec_generator = nullptr; + + BitrateStatisticsObserver* send_bitrate_observer = nullptr; + SendSideDelayObserver* send_side_delay_observer = nullptr; + RtcEventLog* event_log = nullptr; + SendPacketObserver* send_packet_observer = nullptr; + RateLimiter* retransmission_rate_limiter = nullptr; + StreamDataCountersCallback* rtp_stats_callback = nullptr; + + int rtcp_report_interval_ms = 0; + + // Update network2 instead of pacer_exit field of video timing extension. + bool populate_network2_timestamp = false; + + rtc::scoped_refptr<FrameTransformerInterface> frame_transformer; + + // E2EE Custom Video Frame Encryption + FrameEncryptorInterface* frame_encryptor = nullptr; + // Require all outgoing frames to be encrypted with a FrameEncryptor. + bool require_frame_encryption = false; + + // Corresponds to extmap-allow-mixed in SDP negotiation. + bool extmap_allow_mixed = false; + + // If true, the RTP sender will always annotate outgoing packets with + // MID and RID header extensions, if provided and negotiated. + // If false, the RTP sender will stop sending MID and RID header extensions, + // when it knows that the receiver is ready to demux based on SSRC. This is + // done by RTCP RR acking. + bool always_send_mid_and_rid = false; + + // If set, field trials are read from |field_trials|, otherwise + // defaults to webrtc::FieldTrialBasedConfig. + const WebRtcKeyValueConfig* field_trials = nullptr; + + // SSRCs for media and retransmission, respectively. + // FlexFec SSRC is fetched from |flexfec_sender|. + uint32_t local_media_ssrc = 0; + absl::optional<uint32_t> rtx_send_ssrc; + + bool need_rtp_packet_infos = false; + + // If true, the RTP packet history will select RTX packets based on + // heuristics such as send time, retransmission count etc, in order to + // make padding potentially more useful. + // If false, the last packet will always be picked. This may reduce CPU + // overhead. + bool enable_rtx_padding_prioritization = true; + + private: + RTC_DISALLOW_COPY_AND_ASSIGN(Configuration); + }; + + // ************************************************************************** + // Receiver functions + // ************************************************************************** + + virtual void IncomingRtcpPacket(const uint8_t* incoming_packet, + size_t incoming_packet_length) = 0; + + virtual void SetRemoteSSRC(uint32_t ssrc) = 0; + + // ************************************************************************** + // Sender + // ************************************************************************** + + // Sets the maximum size of an RTP packet, including RTP headers. + virtual void SetMaxRtpPacketSize(size_t size) = 0; + + // Returns max RTP packet size. Takes into account RTP headers and + // FEC/ULP/RED overhead (when FEC is enabled). + virtual size_t MaxRtpPacketSize() const = 0; + + virtual void RegisterSendPayloadFrequency(int payload_type, + int payload_frequency) = 0; + + // Unregisters a send payload. + // |payload_type| - payload type of codec + // Returns -1 on failure else 0. + virtual int32_t DeRegisterSendPayload(int8_t payload_type) = 0; + + virtual void SetExtmapAllowMixed(bool extmap_allow_mixed) = 0; + + // Register extension by uri, triggers CHECK on falure. + virtual void RegisterRtpHeaderExtension(absl::string_view uri, int id) = 0; + + virtual int32_t DeregisterSendRtpHeaderExtension(RTPExtensionType type) = 0; + virtual void DeregisterSendRtpHeaderExtension(absl::string_view uri) = 0; + + // Returns true if RTP module is send media, and any of the extensions + // required for bandwidth estimation is registered. + virtual bool SupportsPadding() const = 0; + // Same as SupportsPadding(), but additionally requires that + // SetRtxSendStatus() has been called with the kRtxRedundantPayloads option + // enabled. + virtual bool SupportsRtxPayloadPadding() const = 0; + + // Returns start timestamp. + virtual uint32_t StartTimestamp() const = 0; + + // Sets start timestamp. Start timestamp is set to a random value if this + // function is never called. + virtual void SetStartTimestamp(uint32_t timestamp) = 0; + + // Returns SequenceNumber. + virtual uint16_t SequenceNumber() const = 0; + + // Sets SequenceNumber, default is a random number. + virtual void SetSequenceNumber(uint16_t seq) = 0; + + virtual void SetRtpState(const RtpState& rtp_state) = 0; + virtual void SetRtxState(const RtpState& rtp_state) = 0; + virtual RtpState GetRtpState() const = 0; + virtual RtpState GetRtxState() const = 0; + + // Returns SSRC. + virtual uint32_t SSRC() const = 0; + + // Sets the value for sending in the RID (and Repaired) RTP header extension. + // RIDs are used to identify an RTP stream if SSRCs are not negotiated. + // If the RID and Repaired RID extensions are not registered, the RID will + // not be sent. + virtual void SetRid(const std::string& rid) = 0; + + // Sets the value for sending in the MID RTP header extension. + // The MID RTP header extension should be registered for this to do anything. + // Once set, this value can not be changed or removed. + virtual void SetMid(const std::string& mid) = 0; + + // Sets CSRC. + // |csrcs| - vector of CSRCs + virtual void SetCsrcs(const std::vector<uint32_t>& csrcs) = 0; + + // Turns on/off sending RTX (RFC 4588). The modes can be set as a combination + // of values of the enumerator RtxMode. + virtual void SetRtxSendStatus(int modes) = 0; + + // Returns status of sending RTX (RFC 4588). The returned value can be + // a combination of values of the enumerator RtxMode. + virtual int RtxSendStatus() const = 0; + + // Returns the SSRC used for RTX if set, otherwise a nullopt. + virtual absl::optional<uint32_t> RtxSsrc() const = 0; + + // Sets the payload type to use when sending RTX packets. Note that this + // doesn't enable RTX, only the payload type is set. + virtual void SetRtxSendPayloadType(int payload_type, + int associated_payload_type) = 0; + + // Returns the FlexFEC SSRC, if there is one. + virtual absl::optional<uint32_t> FlexfecSsrc() const = 0; + + // Sets sending status. Sends kRtcpByeCode when going from true to false. + // Returns -1 on failure else 0. + virtual int32_t SetSendingStatus(bool sending) = 0; + + // Returns current sending status. + virtual bool Sending() const = 0; + + // Starts/Stops media packets. On by default. + virtual void SetSendingMediaStatus(bool sending) = 0; + + // Returns current media sending status. + virtual bool SendingMedia() const = 0; + + // Returns whether audio is configured (i.e. Configuration::audio = true). + virtual bool IsAudioConfigured() const = 0; + + // Indicate that the packets sent by this module should be counted towards the + // bitrate estimate since the stream participates in the bitrate allocation. + virtual void SetAsPartOfAllocation(bool part_of_allocation) = 0; + + // TODO(sprang): Remove when all call sites have been moved to + // GetSendRates(). Fetches the current send bitrates in bits/s. + virtual void BitrateSent(uint32_t* total_rate, + uint32_t* video_rate, + uint32_t* fec_rate, + uint32_t* nack_rate) const = 0; + + // Returns bitrate sent (post-pacing) per packet type. + virtual RtpSendRates GetSendRates() const = 0; + + virtual RTPSender* RtpSender() = 0; + virtual const RTPSender* RtpSender() const = 0; + + // Record that a frame is about to be sent. Returns true on success, and false + // if the module isn't ready to send. + virtual bool OnSendingRtpFrame(uint32_t timestamp, + int64_t capture_time_ms, + int payload_type, + bool force_sender_report) = 0; + + // Try to send the provided packet. Returns true iff packet matches any of + // the SSRCs for this module (media/rtx/fec etc) and was forwarded to the + // transport. + virtual bool TrySendPacket(RtpPacketToSend* packet, + const PacedPacketInfo& pacing_info) = 0; + + virtual void OnPacketsAcknowledged( + rtc::ArrayView<const uint16_t> sequence_numbers) = 0; + + virtual std::vector<std::unique_ptr<RtpPacketToSend>> GeneratePadding( + size_t target_size_bytes) = 0; + + virtual std::vector<RtpSequenceNumberMap::Info> GetSentRtpPacketInfos( + rtc::ArrayView<const uint16_t> sequence_numbers) const = 0; + + // Returns an expected per packet overhead representing the main RTP header, + // any CSRCs, and the registered header extensions that are expected on all + // packets (i.e. disregarding things like abs capture time which is only + // populated on a subset of packets, but counting MID/RID type extensions + // when we expect to send them). + virtual size_t ExpectedPerPacketOverhead() const = 0; + + // ************************************************************************** + // RTCP + // ************************************************************************** + + // Returns RTCP status. + virtual RtcpMode RTCP() const = 0; + + // Sets RTCP status i.e on(compound or non-compound)/off. + // |method| - RTCP method to use. + virtual void SetRTCPStatus(RtcpMode method) = 0; + + // Sets RTCP CName (i.e unique identifier). + // Returns -1 on failure else 0. + virtual int32_t SetCNAME(const char* cname) = 0; + + // Returns remote NTP. + // Returns -1 on failure else 0. + virtual int32_t RemoteNTP(uint32_t* received_ntp_secs, + uint32_t* received_ntp_frac, + uint32_t* rtcp_arrival_time_secs, + uint32_t* rtcp_arrival_time_frac, + uint32_t* rtcp_timestamp) const = 0; + + // Returns current RTT (round-trip time) estimate. + // Returns -1 on failure else 0. + virtual int32_t RTT(uint32_t remote_ssrc, + int64_t* rtt, + int64_t* avg_rtt, + int64_t* min_rtt, + int64_t* max_rtt) const = 0; + + // Returns the estimated RTT, with fallback to a default value. + virtual int64_t ExpectedRetransmissionTimeMs() const = 0; + + // Forces a send of a RTCP packet. Periodic SR and RR are triggered via the + // process function. + // Returns -1 on failure else 0. + virtual int32_t SendRTCP(RTCPPacketType rtcp_packet_type) = 0; + + // Returns send statistics for the RTP and RTX stream. + virtual void GetSendStreamDataCounters( + StreamDataCounters* rtp_counters, + StreamDataCounters* rtx_counters) const = 0; + + // Returns received RTCP report block. + // Returns -1 on failure else 0. + // TODO(https://crbug.com/webrtc/10678): Remove this in favor of + // GetLatestReportBlockData(). + virtual int32_t RemoteRTCPStat( + std::vector<RTCPReportBlock>* receive_blocks) const = 0; + // A snapshot of Report Blocks with additional data of interest to statistics. + // Within this list, the sender-source SSRC pair is unique and per-pair the + // ReportBlockData represents the latest Report Block that was received for + // that pair. + virtual std::vector<ReportBlockData> GetLatestReportBlockData() const = 0; + + // (XR) Sets Receiver Reference Time Report (RTTR) status. + virtual void SetRtcpXrRrtrStatus(bool enable) = 0; + + // Returns current Receiver Reference Time Report (RTTR) status. + virtual bool RtcpXrRrtrStatus() const = 0; + + // (REMB) Receiver Estimated Max Bitrate. + // Schedules sending REMB on next and following sender/receiver reports. + void SetRemb(int64_t bitrate_bps, std::vector<uint32_t> ssrcs) override = 0; + // Stops sending REMB on next and following sender/receiver reports. + void UnsetRemb() override = 0; + + // (NACK) + + // Sends a Negative acknowledgement packet. + // Returns -1 on failure else 0. + // TODO(philipel): Deprecate this and start using SendNack instead, mostly + // because we want a function that actually send NACK for the specified + // packets. + virtual int32_t SendNACK(const uint16_t* nack_list, uint16_t size) = 0; + + // Sends NACK for the packets specified. + // Note: This assumes the caller keeps track of timing and doesn't rely on + // the RTP module to do this. + virtual void SendNack(const std::vector<uint16_t>& sequence_numbers) = 0; + + // Store the sent packets, needed to answer to a Negative acknowledgment + // requests. + virtual void SetStorePacketsStatus(bool enable, uint16_t numberToStore) = 0; + + // Returns true if the module is configured to store packets. + virtual bool StorePackets() const = 0; + + virtual void SetVideoBitrateAllocation( + const VideoBitrateAllocation& bitrate) = 0; + + // ************************************************************************** + // Video + // ************************************************************************** + + // Requests new key frame. + // using PLI, https://tools.ietf.org/html/rfc4585#section-6.3.1.1 + void SendPictureLossIndication() { SendRTCP(kRtcpPli); } + // using FIR, https://tools.ietf.org/html/rfc5104#section-4.3.1.2 + void SendFullIntraRequest() { SendRTCP(kRtcpFir); } + + // Sends a LossNotification RTCP message. + // Returns -1 on failure else 0. + virtual int32_t SendLossNotification(uint16_t last_decoded_seq_num, + uint16_t last_received_seq_num, + bool decodability_flag, + bool buffering_allowed) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_RTP_RTCP_INTERFACE_H_ diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.cc index 3023e595578..af2752d6e3c 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.cc @@ -108,7 +108,6 @@ bool IsNonVolatile(RTPExtensionType type) { case kRtpExtensionAbsoluteSendTime: case kRtpExtensionTransportSequenceNumber: case kRtpExtensionTransportSequenceNumber02: - case kRtpExtensionFrameMarking: case kRtpExtensionRtpStreamId: case kRtpExtensionMid: case kRtpExtensionGenericFrameDescriptor00: @@ -154,7 +153,7 @@ double GetMaxPaddingSizeFactor(const WebRtcKeyValueConfig* field_trials) { } // namespace -RTPSender::RTPSender(const RtpRtcp::Configuration& config, +RTPSender::RTPSender(const RtpRtcpInterface::Configuration& config, RtpPacketHistory* packet_history, RtpPacketSender* packet_sender) : clock_(config.clock), @@ -431,7 +430,7 @@ std::vector<std::unique_ptr<RtpPacketToSend>> RTPSender::GeneratePadding( size_t padding_bytes_in_packet; const size_t max_payload_size = - max_packet_size_ - FecOrPaddingPacketMaxRtpHeaderLength(); + max_packet_size_ - max_padding_fec_packet_header_; if (audio_configured_) { // Allow smaller padding packets for audio. padding_bytes_in_packet = rtc::SafeClamp<size_t>( diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.h index a14c3ae1a82..dd291f8d59a 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender.h @@ -25,10 +25,10 @@ #include "modules/rtp_rtcp/include/flexfec_sender.h" #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" #include "modules/rtp_rtcp/include/rtp_packet_sender.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_history.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/critical_section.h" #include "rtc_base/deprecation.h" @@ -45,105 +45,133 @@ class RtpPacketToSend; class RTPSender { public: - RTPSender(const RtpRtcp::Configuration& config, + RTPSender(const RtpRtcpInterface::Configuration& config, RtpPacketHistory* packet_history, RtpPacketSender* packet_sender); ~RTPSender(); - void SetSendingMediaStatus(bool enabled); - bool SendingMedia() const; - bool IsAudioConfigured() const; + void SetSendingMediaStatus(bool enabled) RTC_LOCKS_EXCLUDED(send_critsect_); + bool SendingMedia() const RTC_LOCKS_EXCLUDED(send_critsect_); + bool IsAudioConfigured() const RTC_LOCKS_EXCLUDED(send_critsect_); - uint32_t TimestampOffset() const; - void SetTimestampOffset(uint32_t timestamp); + uint32_t TimestampOffset() const RTC_LOCKS_EXCLUDED(send_critsect_); + void SetTimestampOffset(uint32_t timestamp) + RTC_LOCKS_EXCLUDED(send_critsect_); - void SetRid(const std::string& rid); + void SetRid(const std::string& rid) RTC_LOCKS_EXCLUDED(send_critsect_); - void SetMid(const std::string& mid); + void SetMid(const std::string& mid) RTC_LOCKS_EXCLUDED(send_critsect_); - uint16_t SequenceNumber() const; - void SetSequenceNumber(uint16_t seq); + uint16_t SequenceNumber() const RTC_LOCKS_EXCLUDED(send_critsect_); + void SetSequenceNumber(uint16_t seq) RTC_LOCKS_EXCLUDED(send_critsect_); - void SetCsrcs(const std::vector<uint32_t>& csrcs); + void SetCsrcs(const std::vector<uint32_t>& csrcs) + RTC_LOCKS_EXCLUDED(send_critsect_); - void SetMaxRtpPacketSize(size_t max_packet_size); + void SetMaxRtpPacketSize(size_t max_packet_size) + RTC_LOCKS_EXCLUDED(send_critsect_); - void SetExtmapAllowMixed(bool extmap_allow_mixed); + void SetExtmapAllowMixed(bool extmap_allow_mixed) + RTC_LOCKS_EXCLUDED(send_critsect_); // RTP header extension - int32_t RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id); - bool RegisterRtpHeaderExtension(absl::string_view uri, int id); - bool IsRtpHeaderExtensionRegistered(RTPExtensionType type) const; - int32_t DeregisterRtpHeaderExtension(RTPExtensionType type); - void DeregisterRtpHeaderExtension(absl::string_view uri); - - bool SupportsPadding() const; - bool SupportsRtxPayloadPadding() const; + int32_t RegisterRtpHeaderExtension(RTPExtensionType type, uint8_t id) + RTC_LOCKS_EXCLUDED(send_critsect_); + bool RegisterRtpHeaderExtension(absl::string_view uri, int id) + RTC_LOCKS_EXCLUDED(send_critsect_); + bool IsRtpHeaderExtensionRegistered(RTPExtensionType type) const + RTC_LOCKS_EXCLUDED(send_critsect_); + int32_t DeregisterRtpHeaderExtension(RTPExtensionType type) + RTC_LOCKS_EXCLUDED(send_critsect_); + void DeregisterRtpHeaderExtension(absl::string_view uri) + RTC_LOCKS_EXCLUDED(send_critsect_); + + bool SupportsPadding() const RTC_LOCKS_EXCLUDED(send_critsect_); + bool SupportsRtxPayloadPadding() const RTC_LOCKS_EXCLUDED(send_critsect_); std::vector<std::unique_ptr<RtpPacketToSend>> GeneratePadding( size_t target_size_bytes, - bool media_has_been_sent); + bool media_has_been_sent) RTC_LOCKS_EXCLUDED(send_critsect_); // NACK. void OnReceivedNack(const std::vector<uint16_t>& nack_sequence_numbers, - int64_t avg_rtt); + int64_t avg_rtt) RTC_LOCKS_EXCLUDED(send_critsect_); - int32_t ReSendPacket(uint16_t packet_id); + int32_t ReSendPacket(uint16_t packet_id) RTC_LOCKS_EXCLUDED(send_critsect_); // ACK. - void OnReceivedAckOnSsrc(int64_t extended_highest_sequence_number); - void OnReceivedAckOnRtxSsrc(int64_t extended_highest_sequence_number); + void OnReceivedAckOnSsrc(int64_t extended_highest_sequence_number) + RTC_LOCKS_EXCLUDED(send_critsect_); + void OnReceivedAckOnRtxSsrc(int64_t extended_highest_sequence_number) + RTC_LOCKS_EXCLUDED(send_critsect_); // RTX. - void SetRtxStatus(int mode); - int RtxStatus() const; - absl::optional<uint32_t> RtxSsrc() const { return rtx_ssrc_; } + void SetRtxStatus(int mode) RTC_LOCKS_EXCLUDED(send_critsect_); + int RtxStatus() const RTC_LOCKS_EXCLUDED(send_critsect_); + absl::optional<uint32_t> RtxSsrc() const RTC_LOCKS_EXCLUDED(send_critsect_) { + return rtx_ssrc_; + } - void SetRtxPayloadType(int payload_type, int associated_payload_type); + void SetRtxPayloadType(int payload_type, int associated_payload_type) + RTC_LOCKS_EXCLUDED(send_critsect_); // Size info for header extensions used by FEC packets. - static rtc::ArrayView<const RtpExtensionSize> FecExtensionSizes(); + static rtc::ArrayView<const RtpExtensionSize> FecExtensionSizes() + RTC_LOCKS_EXCLUDED(send_critsect_); // Size info for header extensions used by video packets. - static rtc::ArrayView<const RtpExtensionSize> VideoExtensionSizes(); + static rtc::ArrayView<const RtpExtensionSize> VideoExtensionSizes() + RTC_LOCKS_EXCLUDED(send_critsect_); // Size info for header extensions used by audio packets. - static rtc::ArrayView<const RtpExtensionSize> AudioExtensionSizes(); + static rtc::ArrayView<const RtpExtensionSize> AudioExtensionSizes() + RTC_LOCKS_EXCLUDED(send_critsect_); // Create empty packet, fills ssrc, csrcs and reserve place for header // extensions RtpSender updates before sending. - std::unique_ptr<RtpPacketToSend> AllocatePacket() const; + std::unique_ptr<RtpPacketToSend> AllocatePacket() const + RTC_LOCKS_EXCLUDED(send_critsect_); // Allocate sequence number for provided packet. // Save packet's fields to generate padding that doesn't break media stream. // Return false if sending was turned off. - bool AssignSequenceNumber(RtpPacketToSend* packet); + bool AssignSequenceNumber(RtpPacketToSend* packet) + RTC_LOCKS_EXCLUDED(send_critsect_); // Maximum header overhead per fec/padding packet. - size_t FecOrPaddingPacketMaxRtpHeaderLength() const; + size_t FecOrPaddingPacketMaxRtpHeaderLength() const + RTC_LOCKS_EXCLUDED(send_critsect_); // Expected header overhead per media packet. - size_t ExpectedPerPacketOverhead() const; - uint16_t AllocateSequenceNumber(uint16_t packets_to_send); + size_t ExpectedPerPacketOverhead() const RTC_LOCKS_EXCLUDED(send_critsect_); + uint16_t AllocateSequenceNumber(uint16_t packets_to_send) + RTC_LOCKS_EXCLUDED(send_critsect_); // Including RTP headers. - size_t MaxRtpPacketSize() const; + size_t MaxRtpPacketSize() const RTC_LOCKS_EXCLUDED(send_critsect_); - uint32_t SSRC() const { return ssrc_; } + uint32_t SSRC() const RTC_LOCKS_EXCLUDED(send_critsect_) { return ssrc_; } - absl::optional<uint32_t> FlexfecSsrc() const { return flexfec_ssrc_; } + absl::optional<uint32_t> FlexfecSsrc() const + RTC_LOCKS_EXCLUDED(send_critsect_) { + return flexfec_ssrc_; + } // Sends packet to |transport_| or to the pacer, depending on configuration. // TODO(bugs.webrtc.org/XXX): Remove in favor of EnqueuePackets(). - bool SendToNetwork(std::unique_ptr<RtpPacketToSend> packet); + bool SendToNetwork(std::unique_ptr<RtpPacketToSend> packet) + RTC_LOCKS_EXCLUDED(send_critsect_); // Pass a set of packets to RtpPacketSender instance, for paced or immediate // sending to the network. - void EnqueuePackets(std::vector<std::unique_ptr<RtpPacketToSend>> packets); + void EnqueuePackets(std::vector<std::unique_ptr<RtpPacketToSend>> packets) + RTC_LOCKS_EXCLUDED(send_critsect_); - void SetRtpState(const RtpState& rtp_state); - RtpState GetRtpState() const; - void SetRtxRtpState(const RtpState& rtp_state); - RtpState GetRtxRtpState() const; + void SetRtpState(const RtpState& rtp_state) + RTC_LOCKS_EXCLUDED(send_critsect_); + RtpState GetRtpState() const RTC_LOCKS_EXCLUDED(send_critsect_); + void SetRtxRtpState(const RtpState& rtp_state) + RTC_LOCKS_EXCLUDED(send_critsect_); + RtpState GetRtxRtpState() const RTC_LOCKS_EXCLUDED(send_critsect_); - int64_t LastTimestampTimeMs() const; + int64_t LastTimestampTimeMs() const RTC_LOCKS_EXCLUDED(send_critsect_); private: std::unique_ptr<RtpPacketToSend> BuildRtxPacket( diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc index 3e35f42bffe..1583ab04c03 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc @@ -18,6 +18,7 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/time_util.h" #include "test/gmock.h" #include "test/gtest.h" @@ -67,8 +68,8 @@ class RtpSenderAudioTest : public ::testing::Test { public: RtpSenderAudioTest() : fake_clock_(kStartTime), - rtp_module_(RtpRtcp::Create([&] { - RtpRtcp::Configuration config; + rtp_module_(ModuleRtpRtcpImpl2::Create([&] { + RtpRtcpInterface::Configuration config; config.audio = true; config.clock = &fake_clock_; config.outgoing_transport = &transport_; @@ -81,7 +82,7 @@ class RtpSenderAudioTest : public ::testing::Test { SimulatedClock fake_clock_; LoopbackTransportTest transport_; - std::unique_ptr<RtpRtcp> rtp_module_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_module_; RTPSenderAudio rtp_sender_audio_; }; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.cc index 6d5477be213..c309fc3f0dd 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.cc @@ -53,7 +53,7 @@ void RtpSenderEgress::NonPacedPacketSender::EnqueuePackets( } } -RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config, +RtpSenderEgress::RtpSenderEgress(const RtpRtcpInterface::Configuration& config, RtpPacketHistory* packet_history) : ssrc_(config.local_media_ssrc), rtx_ssrc_(config.rtx_send_ssrc), @@ -84,7 +84,9 @@ RtpSenderEgress::RtpSenderEgress(const RtpRtcp::Configuration& config, rtp_sequence_number_map_(need_rtp_packet_infos_ ? std::make_unique<RtpSequenceNumberMap>( kRtpSequenceNumberMapMaxEntries) - : nullptr) {} + : nullptr) { + RTC_DCHECK(TaskQueueBase::Current()); +} void RtpSenderEgress::SendPacket(RtpPacketToSend* packet, const PacedPacketInfo& pacing_info) { diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.h index c9ecde3be8b..a8e033c5bf1 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_egress.h @@ -19,10 +19,10 @@ #include "api/call/transport.h" #include "api/rtc_event_log/rtc_event_log.h" #include "api/units/data_rate.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_packet_history.h" #include "modules/rtp_rtcp/source/rtp_packet_to_send.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" #include "modules/rtp_rtcp/source/rtp_sequence_number_map.h" #include "rtc_base/critical_section.h" #include "rtc_base/rate_statistics.h" @@ -47,7 +47,7 @@ class RtpSenderEgress { RtpSenderEgress* const sender_; }; - RtpSenderEgress(const RtpRtcp::Configuration& config, + RtpSenderEgress(const RtpRtcpInterface::Configuration& config, RtpPacketHistory* packet_history); ~RtpSenderEgress() = default; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc index 65e2e04ef44..12055b5b1ca 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_unittest.cc @@ -149,33 +149,32 @@ class MockRtpPacketPacer : public RtpPacketSender { MockRtpPacketPacer() {} virtual ~MockRtpPacketPacer() {} - MOCK_METHOD1(EnqueuePackets, - void(std::vector<std::unique_ptr<RtpPacketToSend>>)); - - MOCK_METHOD2(CreateProbeCluster, void(int bitrate_bps, int cluster_id)); - - MOCK_METHOD0(Pause, void()); - MOCK_METHOD0(Resume, void()); - MOCK_METHOD1(SetCongestionWindow, - void(absl::optional<int64_t> congestion_window_bytes)); - MOCK_METHOD1(UpdateOutstandingData, void(int64_t outstanding_bytes)); - MOCK_METHOD1(SetAccountForAudioPackets, void(bool account_for_audio)); + MOCK_METHOD(void, + EnqueuePackets, + (std::vector<std::unique_ptr<RtpPacketToSend>>), + (override)); }; class MockSendSideDelayObserver : public SendSideDelayObserver { public: - MOCK_METHOD4(SendSideDelayUpdated, void(int, int, uint64_t, uint32_t)); + MOCK_METHOD(void, + SendSideDelayUpdated, + (int, int, uint64_t, uint32_t), + (override)); }; class MockSendPacketObserver : public SendPacketObserver { public: - MOCK_METHOD3(OnSendPacket, void(uint16_t, int64_t, uint32_t)); + MOCK_METHOD(void, OnSendPacket, (uint16_t, int64_t, uint32_t), (override)); }; class MockTransportFeedbackObserver : public TransportFeedbackObserver { public: - MOCK_METHOD1(OnAddPacket, void(const RtpPacketSendInfo&)); - MOCK_METHOD1(OnTransportFeedback, void(const rtcp::TransportFeedback&)); + MOCK_METHOD(void, OnAddPacket, (const RtpPacketSendInfo&), (override)); + MOCK_METHOD(void, + OnTransportFeedback, + (const rtcp::TransportFeedback&), + (override)); }; class StreamDataTestCallback : public StreamDataCountersCallback { @@ -213,7 +212,7 @@ class StreamDataTestCallback : public StreamDataCountersCallback { // TODO(sprang): Split up unit tests and test these components individually // wherever possible. struct RtpSenderContext { - explicit RtpSenderContext(const RtpRtcp::Configuration& config) + explicit RtpSenderContext(const RtpRtcpInterface::Configuration& config) : packet_history_(config.clock, config.enable_rtx_padding_prioritization), packet_sender_(config, &packet_history_), non_paced_sender_(&packet_sender_), @@ -286,7 +285,7 @@ class RtpSenderTest : public ::testing::TestWithParam<TestConfig> { void SetUpRtpSender(bool pacer, bool populate_network2, bool always_send_mid_and_rid) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -482,7 +481,7 @@ TEST_P(RtpSenderTestWithoutPacer, AssignSequenceNumberMayAllowPaddingOnVideo) { TEST_P(RtpSenderTest, AssignSequenceNumberAllowsPaddingOnAudio) { MockTransport transport; - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.audio = true; config.clock = &fake_clock_; config.outgoing_transport = &transport; @@ -532,7 +531,7 @@ TEST_P(RtpSenderTestWithoutPacer, TransportFeedbackObserverGetsCorrectByteCount) { constexpr size_t kRtpOverheadBytesPerPacket = 12 + 8; - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -567,7 +566,7 @@ TEST_P(RtpSenderTestWithoutPacer, } TEST_P(RtpSenderTestWithoutPacer, SendsPacketsWithTransportSequenceNumber) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -606,7 +605,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendsPacketsWithTransportSequenceNumber) { } TEST_P(RtpSenderTestWithoutPacer, PacketOptionsNoRetransmission) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -661,7 +660,7 @@ TEST_P(RtpSenderTestWithoutPacer, DoesnSetIncludedInAllocationByDefault) { TEST_P(RtpSenderTestWithoutPacer, OnSendSideDelayUpdated) { StrictMock<MockSendSideDelayObserver> send_side_delay_observer_; - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -748,7 +747,7 @@ TEST_P(RtpSenderTestWithoutPacer, OnSendPacketUpdated) { } TEST_P(RtpSenderTest, SendsPacketsWithTransportSequenceNumber) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; @@ -1240,7 +1239,7 @@ TEST_P(RtpSenderTest, SendFlexfecPackets) { nullptr /* rtp_state */, &fake_clock_); // Reset |rtp_sender_| to use FlexFEC. - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; @@ -1329,7 +1328,7 @@ TEST_P(RtpSenderTestWithoutPacer, SendFlexfecPackets) { nullptr /* rtp_state */, &fake_clock_); // Reset |rtp_sender_| to use FlexFEC. - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -1662,7 +1661,7 @@ TEST_P(RtpSenderTest, FecOverheadRate) { nullptr /* rtp_state */, &fake_clock_); // Reset |rtp_sender_| to use FlexFEC. - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.paced_sender = &mock_paced_sender_; @@ -1743,7 +1742,7 @@ TEST_P(RtpSenderTest, BitrateCallbacks) { uint32_t retransmit_bitrate_; } callback; - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -1971,7 +1970,7 @@ TEST_P(RtpSenderTestWithoutPacer, RespectsNackBitrateLimit) { } TEST_P(RtpSenderTest, UpdatingCsrcsUpdatedOverhead) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -1987,7 +1986,7 @@ TEST_P(RtpSenderTest, UpdatingCsrcsUpdatedOverhead) { } TEST_P(RtpSenderTest, OnOverheadChanged) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -2006,7 +2005,7 @@ TEST_P(RtpSenderTest, OnOverheadChanged) { } TEST_P(RtpSenderTest, CountMidOnlyUntilAcked) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -2033,7 +2032,7 @@ TEST_P(RtpSenderTest, CountMidOnlyUntilAcked) { } TEST_P(RtpSenderTest, DontCountVolatileExtensionsIntoOverhead) { - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; @@ -2251,7 +2250,7 @@ TEST_P(RtpSenderTest, SendPacketUpdatesStats) { StrictMock<MockSendSideDelayObserver> send_side_delay_observer; - RtpRtcp::Configuration config; + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.local_media_ssrc = kSsrc; diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc index 02c4eb8d286..58a86996886 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -145,7 +145,7 @@ RTPSenderVideo::RTPSenderVideo(const Config& config) this, config.frame_transformer, rtp_sender_->SSRC(), - config.worker_queue) + config.send_transport_queue) : nullptr) { if (frame_transformer_delegate_) frame_transformer_delegate_->Init(); @@ -252,8 +252,6 @@ void RTPSenderVideo::SetVideoStructureUnderLock( video_structure_ = std::make_unique<FrameDependencyStructure>(*video_structure); video_structure_->structure_id = structure_id; - // TODO(bugs.webrtc.org/10342): Support chains. - video_structure_->num_chains = 0; } void RTPSenderVideo::AddRtpHeaderExtensions( @@ -314,14 +312,6 @@ void RTPSenderVideo::AddRtpHeaderExtensions( packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time); } - if (video_header.codec == kVideoCodecH264 && - video_header.frame_marking.temporal_id != kNoTemporalIdx) { - FrameMarking frame_marking = video_header.frame_marking; - frame_marking.start_of_frame = first_packet; - frame_marking.end_of_frame = last_packet; - packet->SetExtension<FrameMarkingExtension>(frame_marking); - } - if (video_header.generic) { bool extension_is_set = false; if (video_structure_ != nullptr) { @@ -337,6 +327,8 @@ void RTPSenderVideo::AddRtpHeaderExtensions( descriptor.frame_dependencies.frame_diffs.push_back( video_header.generic->frame_id - dep); } + descriptor.frame_dependencies.chain_diffs = + video_header.generic->chain_diffs; descriptor.frame_dependencies.decode_target_indications = video_header.generic->decode_target_indications; RTC_DCHECK_EQ( @@ -736,12 +728,7 @@ uint8_t RTPSenderVideo::GetTemporalId(const RTPVideoHeader& header) { } uint8_t operator()(const absl::monostate&) { return kNoTemporalIdx; } }; - switch (header.codec) { - case kVideoCodecH264: - return header.frame_marking.temporal_id; - default: - return absl::visit(TemporalIdGetter(), header.video_type_header); - } + return absl::visit(TemporalIdGetter(), header.video_type_header); } bool RTPSenderVideo::UpdateConditionalRetransmit( diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h index 216f16faf6a..699734efa31 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video.h @@ -82,7 +82,7 @@ class RTPSenderVideo { absl::optional<int> red_payload_type; const WebRtcKeyValueConfig* field_trials = nullptr; rtc::scoped_refptr<FrameTransformerInterface> frame_transformer; - TaskQueueBase* worker_queue = nullptr; + TaskQueueBase* send_transport_queue = nullptr; }; explicit RTPSenderVideo(const Config& config); diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 51fbdb01655..a4cacef520f 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -43,6 +43,7 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { uint32_t ssrc) : encoded_data_(encoded_image.GetEncodedData()), header_(video_header), + metadata_(header_), frame_type_(encoded_image._frameType), payload_type_(payload_type), codec_type_(codec_type), @@ -75,6 +76,8 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { return RtpDescriptorAuthentication(header_); } + const VideoFrameMetadata& GetMetadata() const override { return metadata_; } + const RTPVideoHeader& GetHeader() const { return header_; } int GetPayloadType() const { return payload_type_; } absl::optional<VideoCodecType> GetCodecType() const { return codec_type_; } @@ -91,6 +94,7 @@ class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { private: rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data_; const RTPVideoHeader header_; + const VideoFrameMetadata metadata_; const VideoFrameType frame_type_; const int payload_type_; const absl::optional<VideoCodecType> codec_type_ = absl::nullopt; @@ -106,11 +110,11 @@ RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate( RTPSenderVideo* sender, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, uint32_t ssrc, - TaskQueueBase* worker_queue) + TaskQueueBase* send_transport_queue) : sender_(sender), frame_transformer_(std::move(frame_transformer)), ssrc_(ssrc), - worker_queue_(worker_queue) {} + send_transport_queue_(send_transport_queue) {} void RTPSenderVideoFrameTransformerDelegate::Init() { frame_transformer_->RegisterTransformedFrameSinkCallback( @@ -129,9 +133,9 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( // Save the current task queue to post the transformed frame for sending // once it is transformed. When there is no current task queue, i.e. // encoding is done on an external thread (for example in the case of - // hardware encoders), use the worker queue instead. + // hardware encoders), use the send transport queue instead. TaskQueueBase* current = TaskQueueBase::Current(); - encoder_queue_ = current ? current : worker_queue_; + encoder_queue_ = current ? current : send_transport_queue_; } frame_transformer_->Transform(std::make_unique<TransformableVideoSenderFrame>( encoded_image, video_header, payload_type, codec_type, rtp_timestamp, diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index bea5ba7b65a..5beba7770b5 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -31,7 +31,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { RTPSenderVideo* sender, rtc::scoped_refptr<FrameTransformerInterface> frame_transformer, uint32_t ssrc, - TaskQueueBase* worker_queue); + TaskQueueBase* send_transport_queue); void Init(); @@ -70,7 +70,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_; const uint32_t ssrc_; TaskQueueBase* encoder_queue_ = nullptr; - TaskQueueBase* worker_queue_; + TaskQueueBase* send_transport_queue_; }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index 21c4da05abd..32e138f8405 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -15,6 +15,7 @@ #include <utility> #include <vector> +#include "absl/memory/memory.h" #include "api/test/mock_frame_encryptor.h" #include "api/transport/field_trial_based_config.h" #include "api/transport/rtp/dependency_descriptor.h" @@ -23,7 +24,6 @@ #include "common_video/generic_frame_descriptor/generic_frame_info.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" -#include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" @@ -32,6 +32,7 @@ #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" #include "modules/rtp_rtcp/source/time_util.h" #include "rtc_base/arraysize.h" #include "rtc_base/rate_limiter.h" @@ -45,6 +46,7 @@ namespace webrtc { namespace { using ::testing::_; +using ::testing::ContainerEq; using ::testing::ElementsAre; using ::testing::ElementsAreArray; using ::testing::IsEmpty; @@ -57,7 +59,6 @@ using ::testing::WithArgs; enum : int { // The first valid value is 1. kAbsoluteSendTimeExtensionId = 1, - kFrameMarkingExtensionId, kGenericDescriptorId, kDependencyDescriptorId, kTransmissionTimeOffsetExtensionId, @@ -93,8 +94,6 @@ class LoopbackTransportTest : public webrtc::Transport { kGenericDescriptorId); receivers_extensions_.Register<RtpDependencyDescriptorExtension>( kDependencyDescriptorId); - receivers_extensions_.Register<FrameMarkingExtension>( - kFrameMarkingExtensionId); receivers_extensions_.Register<AbsoluteCaptureTimeExtension>( kAbsoluteCaptureTimeExtensionId); receivers_extensions_.Register<PlayoutDelayLimits>( @@ -167,8 +166,8 @@ class RtpSenderVideoTest : public ::testing::TestWithParam<bool> { : field_trials_(GetParam()), fake_clock_(kStartTime), retransmission_rate_limiter_(&fake_clock_, 1000), - rtp_module_(RtpRtcp::Create([&] { - RtpRtcp::Configuration config; + rtp_module_(ModuleRtpRtcpImpl2::Create([&] { + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -188,12 +187,12 @@ class RtpSenderVideoTest : public ::testing::TestWithParam<bool> { int version); protected: - const RtpRtcp::Configuration config_; + const RtpRtcpInterface::Configuration config_; FieldTrials field_trials_; SimulatedClock fake_clock_; LoopbackTransportTest transport_; RateLimiter retransmission_rate_limiter_; - std::unique_ptr<RtpRtcp> rtp_module_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_module_; TestRtpSenderVideo rtp_sender_video_; }; @@ -289,43 +288,6 @@ TEST_P(RtpSenderVideoTest, DeltaFrameHasCVOWhenNonZero) { EXPECT_EQ(kVideoRotation_90, rotation); } -TEST_P(RtpSenderVideoTest, CheckH264FrameMarking) { - uint8_t kFrame[kMaxPacketLength]; - rtp_module_->RegisterRtpHeaderExtension(FrameMarkingExtension::kUri, - kFrameMarkingExtensionId); - - RTPFragmentationHeader frag; - frag.VerifyAndAllocateFragmentationHeader(1); - frag.fragmentationOffset[0] = 0; - frag.fragmentationLength[0] = sizeof(kFrame); - - RTPVideoHeader hdr; - hdr.video_type_header.emplace<RTPVideoHeaderH264>().packetization_mode = - H264PacketizationMode::NonInterleaved; - hdr.codec = kVideoCodecH264; - hdr.frame_marking.temporal_id = kNoTemporalIdx; - hdr.frame_marking.tl0_pic_idx = 99; - hdr.frame_marking.base_layer_sync = true; - hdr.frame_type = VideoFrameType::kVideoFrameDelta; - rtp_sender_video_.SendVideo(kPayload, kType, kTimestamp, 0, kFrame, &frag, - hdr, kDefaultExpectedRetransmissionTimeMs); - - FrameMarking fm; - EXPECT_FALSE( - transport_.last_sent_packet().GetExtension<FrameMarkingExtension>(&fm)); - - hdr.frame_marking.temporal_id = 0; - hdr.frame_type = VideoFrameType::kVideoFrameDelta; - rtp_sender_video_.SendVideo(kPayload, kType, kTimestamp + 1, 0, kFrame, &frag, - hdr, kDefaultExpectedRetransmissionTimeMs); - - EXPECT_TRUE( - transport_.last_sent_packet().GetExtension<FrameMarkingExtension>(&fm)); - EXPECT_EQ(hdr.frame_marking.temporal_id, fm.temporal_id); - EXPECT_EQ(hdr.frame_marking.tl0_pic_idx, fm.tl0_pic_idx); - EXPECT_EQ(hdr.frame_marking.base_layer_sync, fm.base_layer_sync); -} - // Make sure rotation is parsed correctly when the Camera (C) and Flip (F) bits // are set in the CVO byte. TEST_P(RtpSenderVideoTest, SendVideoWithCameraAndFlipCVO) { @@ -367,7 +329,6 @@ TEST_P(RtpSenderVideoTest, RetransmissionTypesH264) { header.video_type_header.emplace<RTPVideoHeaderH264>().packetization_mode = H264PacketizationMode::NonInterleaved; header.codec = kVideoCodecH264; - header.frame_marking.temporal_id = kNoTemporalIdx; EXPECT_FALSE(rtp_sender_video_.AllowRetransmission( header, kRetransmitOff, kDefaultExpectedRetransmissionTimeMs)); @@ -378,14 +339,6 @@ TEST_P(RtpSenderVideoTest, RetransmissionTypesH264) { EXPECT_TRUE(rtp_sender_video_.AllowRetransmission( header, kConditionallyRetransmitHigherLayers, kDefaultExpectedRetransmissionTimeMs)); - - // Test higher level retransmit. - for (int tid = 0; tid <= kMaxTemporalStreams; ++tid) { - header.frame_marking.temporal_id = tid; - EXPECT_TRUE(rtp_sender_video_.AllowRetransmission( - header, kRetransmitHigherLayers | kRetransmitBaseLayer, - kDefaultExpectedRetransmissionTimeMs)); - } } TEST_P(RtpSenderVideoTest, RetransmissionTypesVP8BaseLayer) { @@ -546,9 +499,9 @@ TEST_P(RtpSenderVideoTest, SendsDependencyDescriptorWhenVideoStructureIsSet) { FrameDependencyStructure video_structure; video_structure.num_decode_targets = 2; video_structure.templates = { - GenericFrameInfo::Builder().S(0).T(0).Dtis("SS").Build(), - GenericFrameInfo::Builder().S(1).T(0).Dtis("-S").Build(), - GenericFrameInfo::Builder().S(1).T(1).Dtis("-D").Build(), + FrameDependencyTemplate().S(0).T(0).Dtis("SS"), + FrameDependencyTemplate().S(1).T(0).Dtis("-S"), + FrameDependencyTemplate().S(1).T(1).Dtis("-D"), }; rtp_sender_video_.SetVideoStructure(&video_structure); @@ -606,6 +559,40 @@ TEST_P(RtpSenderVideoTest, SendsDependencyDescriptorWhenVideoStructureIsSet) { ElementsAre(1, 501)); } +TEST_P(RtpSenderVideoTest, PropagatesChainDiffsIntoDependencyDescriptor) { + const int64_t kFrameId = 100000; + uint8_t kFrame[100]; + rtp_module_->RegisterRtpHeaderExtension( + RtpDependencyDescriptorExtension::kUri, kDependencyDescriptorId); + FrameDependencyStructure video_structure; + video_structure.num_decode_targets = 2; + video_structure.num_chains = 1; + // First decode target is protected by the only chain, second one - is not. + video_structure.decode_target_protected_by_chain = {0, 1}; + video_structure.templates = { + FrameDependencyTemplate().S(0).T(0).Dtis("SS").ChainDiffs({1}), + }; + rtp_sender_video_.SetVideoStructure(&video_structure); + + RTPVideoHeader hdr; + RTPVideoHeader::GenericDescriptorInfo& generic = hdr.generic.emplace(); + generic.frame_id = kFrameId; + generic.decode_target_indications = {DecodeTargetIndication::kSwitch, + DecodeTargetIndication::kSwitch}; + generic.chain_diffs = {2}; + hdr.frame_type = VideoFrameType::kVideoFrameKey; + rtp_sender_video_.SendVideo(kPayload, kType, kTimestamp, 0, kFrame, nullptr, + hdr, kDefaultExpectedRetransmissionTimeMs); + + ASSERT_EQ(transport_.packets_sent(), 1); + DependencyDescriptor descriptor_key; + ASSERT_TRUE(transport_.last_sent_packet() + .GetExtension<RtpDependencyDescriptorExtension>( + nullptr, &descriptor_key)); + EXPECT_THAT(descriptor_key.frame_dependencies.chain_diffs, + ContainerEq(generic.chain_diffs)); +} + TEST_P(RtpSenderVideoTest, SetDiffentVideoStructureAvoidsCollisionWithThePreviousStructure) { const int64_t kFrameId = 100000; @@ -615,14 +602,14 @@ TEST_P(RtpSenderVideoTest, FrameDependencyStructure video_structure1; video_structure1.num_decode_targets = 2; video_structure1.templates = { - GenericFrameInfo::Builder().S(0).T(0).Dtis("SS").Build(), - GenericFrameInfo::Builder().S(0).T(1).Dtis("D-").Build(), + FrameDependencyTemplate().S(0).T(0).Dtis("SS"), + FrameDependencyTemplate().S(0).T(1).Dtis("D-"), }; FrameDependencyStructure video_structure2; video_structure2.num_decode_targets = 2; video_structure2.templates = { - GenericFrameInfo::Builder().S(0).T(0).Dtis("SS").Build(), - GenericFrameInfo::Builder().S(0).T(1).Dtis("R-").Build(), + FrameDependencyTemplate().S(0).T(0).Dtis("SS"), + FrameDependencyTemplate().S(0).T(1).Dtis("R-"), }; // Send 1st key frame. @@ -705,7 +692,7 @@ TEST_P(RtpSenderVideoTest, FrameDependencyStructure video_structure; video_structure.num_decode_targets = 1; - video_structure.templates = {GenericFrameInfo::Builder().Dtis("S").Build()}; + video_structure.templates = {FrameDependencyTemplate().Dtis("S")}; rtp_sender_video.SetVideoStructure(&video_structure); // Send key frame. @@ -884,8 +871,8 @@ class RtpSenderVideoWithFrameTransformerTest : public ::testing::Test { RtpSenderVideoWithFrameTransformerTest() : fake_clock_(kStartTime), retransmission_rate_limiter_(&fake_clock_, 1000), - rtp_module_(RtpRtcp::Create([&] { - RtpRtcp::Configuration config; + rtp_module_(ModuleRtpRtcpImpl2::Create([&] { + RtpRtcpInterface::Configuration config; config.clock = &fake_clock_; config.outgoing_transport = &transport_; config.retransmission_rate_limiter = &retransmission_rate_limiter_; @@ -912,7 +899,7 @@ class RtpSenderVideoWithFrameTransformerTest : public ::testing::Test { SimulatedClock fake_clock_; LoopbackTransportTest transport_; RateLimiter retransmission_rate_limiter_; - std::unique_ptr<RtpRtcp> rtp_module_; + std::unique_ptr<ModuleRtpRtcpImpl2> rtp_module_; }; std::unique_ptr<EncodedImage> CreateDefaultEncodedImage() { @@ -989,5 +976,47 @@ TEST_F(RtpSenderVideoWithFrameTransformerTest, OnTransformedFrameSendsVideo) { EXPECT_EQ(transport_.packets_sent(), 1); } +TEST_F(RtpSenderVideoWithFrameTransformerTest, + TransformableFrameMetadataHasCorrectValue) { + rtc::scoped_refptr<MockFrameTransformer> mock_frame_transformer = + new rtc::RefCountedObject<NiceMock<MockFrameTransformer>>(); + std::unique_ptr<RTPSenderVideo> rtp_sender_video = + CreateSenderWithFrameTransformer(mock_frame_transformer); + auto encoded_image = CreateDefaultEncodedImage(); + RTPVideoHeader video_header; + video_header.width = 1280u; + video_header.height = 720u; + RTPVideoHeader::GenericDescriptorInfo& generic = + video_header.generic.emplace(); + generic.frame_id = 10; + generic.temporal_index = 3; + generic.spatial_index = 2; + generic.decode_target_indications = {DecodeTargetIndication::kSwitch}; + generic.dependencies = {5}; + + // Check that the transformable frame passed to the frame transformer has the + // correct metadata. + EXPECT_CALL(*mock_frame_transformer, Transform) + .WillOnce( + [](std::unique_ptr<TransformableFrameInterface> transformable_frame) { + auto frame = + absl::WrapUnique(static_cast<TransformableVideoFrameInterface*>( + transformable_frame.release())); + ASSERT_TRUE(frame); + auto metadata = frame->GetMetadata(); + EXPECT_EQ(metadata.GetWidth(), 1280u); + EXPECT_EQ(metadata.GetHeight(), 720u); + EXPECT_EQ(metadata.GetFrameId(), 10); + EXPECT_EQ(metadata.GetTemporalIndex(), 3); + EXPECT_EQ(metadata.GetSpatialIndex(), 2); + EXPECT_THAT(metadata.GetFrameDependencies(), ElementsAre(5)); + EXPECT_THAT(metadata.GetDecodeTargetIndications(), + ElementsAre(DecodeTargetIndication::kSwitch)); + }); + rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp, + *encoded_image, nullptr, video_header, + kDefaultExpectedRetransmissionTimeMs); +} + } // namespace } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_utility.cc b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_utility.cc index f76d7d0f0b8..c25fd96fa5b 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_utility.cc +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_utility.cc @@ -17,7 +17,6 @@ #include "api/array_view.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" @@ -245,10 +244,6 @@ bool RtpHeaderParser::Parse(RTPHeader* header, header->extension.has_video_timing = false; header->extension.video_timing = {0u, 0u, 0u, 0u, 0u, 0u, false}; - header->extension.has_frame_marking = false; - header->extension.frame_marking = {false, false, false, false, - false, kNoTemporalIdx, 0, 0}; - if (X) { /* RTP header extension, RFC 3550. 0 1 2 3 @@ -497,15 +492,6 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( &header->extension.video_timing); break; } - case kRtpExtensionFrameMarking: { - if (!FrameMarkingExtension::Parse(rtc::MakeArrayView(ptr, len + 1), - &header->extension.frame_marking)) { - RTC_LOG(LS_WARNING) << "Incorrect frame marking len: " << len; - return; - } - header->extension.has_frame_marking = true; - break; - } case kRtpExtensionRtpStreamId: { std::string name(reinterpret_cast<const char*>(ptr), len + 1); if (IsLegalRsidName(name)) { diff --git a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_video_header.h b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_video_header.h index 7071463be44..514340add63 100644 --- a/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_video_header.h +++ b/chromium/third_party/webrtc/modules/rtp_rtcp/source/rtp_video_header.h @@ -19,11 +19,10 @@ #include "api/video/color_space.h" #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" -#include "api/video/video_frame_marking.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" -#include "common_types.h" // NOLINT(build/include) +#include "common_types.h" // NOLINT(build/include_directory) #include "modules/video_coding/codecs/h264/include/h264_globals.h" #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" @@ -53,6 +52,7 @@ struct RTPVideoHeader { int temporal_index = 0; absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications; absl::InlinedVector<int64_t, 5> dependencies; + absl::InlinedVector<int, 4> chain_diffs; }; RTPVideoHeader(); @@ -74,7 +74,6 @@ struct RTPVideoHeader { PlayoutDelay playout_delay = {-1, -1}; VideoSendTiming video_timing; - FrameMarking frame_marking = {false, false, false, false, false, 0xFF, 0, 0}; absl::optional<ColorSpace> color_space; RTPVideoTypeHeader video_type_header; }; diff --git a/chromium/third_party/webrtc/modules/utility/source/process_thread_impl_unittest.cc b/chromium/third_party/webrtc/modules/utility/source/process_thread_impl_unittest.cc index 6f765369f4a..1fef0b67401 100644 --- a/chromium/third_party/webrtc/modules/utility/source/process_thread_impl_unittest.cc +++ b/chromium/third_party/webrtc/modules/utility/source/process_thread_impl_unittest.cc @@ -37,9 +37,9 @@ static const int kEventWaitTimeout = 500; class MockModule : public Module { public: - MOCK_METHOD0(TimeUntilNextProcess, int64_t()); - MOCK_METHOD0(Process, void()); - MOCK_METHOD1(ProcessThreadAttached, void(ProcessThread*)); + MOCK_METHOD(int64_t, TimeUntilNextProcess, (), (override)); + MOCK_METHOD(void, Process, (), (override)); + MOCK_METHOD(void, ProcessThreadAttached, (ProcessThread*), (override)); }; class RaiseEventTask : public QueuedTask { diff --git a/chromium/third_party/webrtc/modules/video_capture/BUILD.gn b/chromium/third_party/webrtc/modules/video_capture/BUILD.gn index 1c024122642..f73472617c4 100644 --- a/chromium/third_party/webrtc/modules/video_capture/BUILD.gn +++ b/chromium/third_party/webrtc/modules/video_capture/BUILD.gn @@ -38,9 +38,9 @@ rtc_library("video_capture_module") { "../../rtc_base:stringutils", "../../rtc_base/synchronization:rw_lock_wrapper", "../../system_wrappers", - "//third_party/abseil-cpp/absl/strings", "//third_party/libyuv", ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } if (!build_with_chromium) { @@ -131,13 +131,13 @@ if (!build_with_chromium) { "../../rtc_base:rtc_base_approved", "../../system_wrappers", "../../test:frame_utils", + "../../test:test_main", "../../test:test_support", "../../test:video_test_common", "../utility", "//testing/gtest", "//third_party/abseil-cpp/absl/memory", ] - deps += [ "../../test:test_main" ] } } } diff --git a/chromium/third_party/webrtc/modules/video_capture/linux/device_info_linux.cc b/chromium/third_party/webrtc/modules/video_capture/linux/device_info_linux.cc index bac5d4078a8..3c8fdd20fad 100644 --- a/chromium/third_party/webrtc/modules/video_capture/linux/device_info_linux.cc +++ b/chromium/third_party/webrtc/modules/video_capture/linux/device_info_linux.cc @@ -47,11 +47,19 @@ uint32_t DeviceInfoLinux::NumberOfDevices() { uint32_t count = 0; char device[20]; int fd = -1; + struct v4l2_capability cap; /* detect /dev/video [0-63]VideoCaptureModule entries */ for (int n = 0; n < 64; n++) { sprintf(device, "/dev/video%d", n); if ((fd = open(device, O_RDONLY)) != -1) { + // query device capabilities and make sure this is a video capture device + if (ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0 || + !(cap.device_caps & V4L2_CAP_VIDEO_CAPTURE)) { + close(fd); + continue; + } + close(fd); count++; } @@ -74,9 +82,16 @@ int32_t DeviceInfoLinux::GetDeviceName(uint32_t deviceNumber, char device[20]; int fd = -1; bool found = false; + struct v4l2_capability cap; for (int n = 0; n < 64; n++) { sprintf(device, "/dev/video%d", n); if ((fd = open(device, O_RDONLY)) != -1) { + // query device capabilities and make sure this is a video capture device + if (ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0 || + !(cap.device_caps & V4L2_CAP_VIDEO_CAPTURE)) { + close(fd); + continue; + } if (count == deviceNumber) { // Found the device found = true; @@ -92,7 +107,6 @@ int32_t DeviceInfoLinux::GetDeviceName(uint32_t deviceNumber, return -1; // query device capabilities - struct v4l2_capability cap; if (ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0) { RTC_LOG(LS_INFO) << "error in querying the device capability for device " << device << ". errno = " << errno; @@ -153,6 +167,11 @@ int32_t DeviceInfoLinux::CreateCapabilityMap(const char* deviceUniqueIdUTF8) { // query device capabilities struct v4l2_capability cap; if (ioctl(fd, VIDIOC_QUERYCAP, &cap) == 0) { + // skip devices without video capture capability + if (!(cap.device_caps & V4L2_CAP_VIDEO_CAPTURE)) { + continue; + } + if (cap.bus_info[0] != 0) { if (strncmp((const char*)cap.bus_info, (const char*)deviceUniqueIdUTF8, strlen((const char*)deviceUniqueIdUTF8)) == diff --git a/chromium/third_party/webrtc/modules/video_coding/BUILD.gn b/chromium/third_party/webrtc/modules/video_coding/BUILD.gn index 41c952694d7..e92649d4e6c 100644 --- a/chromium/third_party/webrtc/modules/video_coding/BUILD.gn +++ b/chromium/third_party/webrtc/modules/video_coding/BUILD.gn @@ -6,6 +6,7 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +import("//third_party/libaom/options.gni") import("../../webrtc.gni") rtc_library("encoded_frame") { @@ -30,11 +31,29 @@ rtc_library("encoded_frame") { "../../rtc_base/experiments:rtt_mult_experiment", "../../rtc_base/system:rtc_export", "../../system_wrappers", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional", "//third_party/abseil-cpp/absl/types:variant", ] } +rtc_library("chain_diff_calculator") { + sources = [ + "chain_diff_calculator.cc", + "chain_diff_calculator.h", + ] + + deps = [ + "../../rtc_base:checks", + "../../rtc_base:logging", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + rtc_library("frame_dependencies_calculator") { sources = [ "frame_dependencies_calculator.cc", @@ -47,6 +66,8 @@ rtc_library("frame_dependencies_calculator") { "../../common_video/generic_frame_descriptor", "../../rtc_base:checks", "../../rtc_base:logging", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/types:optional", @@ -54,12 +75,11 @@ rtc_library("frame_dependencies_calculator") { } rtc_library("nack_module") { - visibility = [ "*" ] sources = [ "histogram.cc", "histogram.h", - "nack_module.cc", - "nack_module.h", + "nack_module2.cc", + "nack_module2.h", ] deps = [ @@ -69,7 +89,11 @@ rtc_library("nack_module") { "../../rtc_base:checks", "../../rtc_base:rtc_base_approved", "../../rtc_base:rtc_numerics", + "../../rtc_base:rtc_task_queue", "../../rtc_base/experiments:field_trial_parser", + "../../rtc_base/synchronization:sequence_checker", + "../../rtc_base/task_utils:pending_task_safety_flag", + "../../rtc_base/task_utils:repeating_task", "../../system_wrappers", "../../system_wrappers:field_trial", "../utility", @@ -91,8 +115,13 @@ rtc_library("video_coding") { "../../system_wrappers:field_trial", "../../system_wrappers:metrics", "../rtp_rtcp:rtp_video_header", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/memory", + "//third_party/abseil-cpp/absl/types:optional", + "//third_party/abseil-cpp/absl/types:variant", ] sources = [ @@ -174,6 +203,7 @@ rtc_library("video_coding") { "../../rtc_base/experiments:min_video_bitrate_experiment", "../../rtc_base/experiments:rate_control_settings", "../../rtc_base/experiments:rtt_mult_experiment", + "../../rtc_base/synchronization:mutex", "../../rtc_base/synchronization:sequence_checker", "../../rtc_base/task_utils:repeating_task", "../../rtc_base/third_party/base64", @@ -181,9 +211,6 @@ rtc_library("video_coding") { "../../system_wrappers", "../rtp_rtcp", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/container:inlined_vector", - "//third_party/abseil-cpp/absl/types:optional", - "//third_party/abseil-cpp/absl/types:variant", ] } @@ -205,8 +232,8 @@ rtc_library("video_codec_interface") { "../../common_video", "../../common_video/generic_frame_descriptor", "../../rtc_base/system:rtc_export", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("video_coding_legacy") { @@ -258,6 +285,8 @@ rtc_library("video_coding_legacy") { "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", "../utility", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", "//third_party/abseil-cpp/absl/types:variant", @@ -329,8 +358,8 @@ rtc_library("video_coding_utility") { "../../rtc_base/task_utils:to_queued_task", "../../system_wrappers:field_trial", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("webrtc_h264") { @@ -363,9 +392,11 @@ rtc_library("webrtc_h264") { "../../rtc_base/system:rtc_export", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", + "//third_party/libyuv", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", - "//third_party/libyuv", ] if (rtc_use_h264) { @@ -449,9 +480,9 @@ rtc_library("webrtc_vp8") { "../../rtc_base/experiments:rate_control_settings", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", - "//third_party/abseil-cpp/absl/types:optional", "//third_party/libyuv", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] if (rtc_build_libvpx) { deps += [ rtc_libvpx_dir ] } @@ -482,8 +513,8 @@ rtc_library("webrtc_vp8_temporal_layers") { "../../rtc_base:rtc_numerics", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } # This target includes VP9 files that may be used for any VP9 codec, internal SW or external HW. @@ -507,8 +538,8 @@ rtc_library("webrtc_vp9_helpers") { "../../rtc_base:checks", "../../rtc_base:logging", "../../rtc_base/experiments:stable_target_rate_experiment", - "//third_party/abseil-cpp/absl/container:inlined_vector", ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_library("webrtc_vp9") { @@ -543,8 +574,8 @@ rtc_library("webrtc_vp9") { "../../rtc_base/experiments:rate_control_settings", "../../system_wrappers:field_trial", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/memory", ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory" ] if (rtc_build_libvpx) { deps += [ rtc_libvpx_dir ] } @@ -595,6 +626,25 @@ if (rtc_include_tests) { } } + rtc_library("encoded_video_frame_producer") { + testonly = true + sources = [ + "codecs/test/encoded_video_frame_producer.cc", + "codecs/test/encoded_video_frame_producer.h", + ] + deps = [ + ":video_codec_interface", + "../../api:create_frame_generator", + "../../api:frame_generator_api", + "../../api/transport/rtp:dependency_descriptor", + "../../api/video:encoded_image", + "../../api/video:video_frame", + "../../api/video:video_frame_type", + "../../api/video_codecs:video_codecs_api", + "../../rtc_base:checks", + ] + } + rtc_library("simulcast_test_fixture_impl") { testonly = true sources = [ @@ -663,9 +713,9 @@ if (rtc_include_tests) { "../../test:video_test_common", "../../test:video_test_support", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/types:optional", "//third_party/libyuv", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } video_coding_modules_tests_resources = [] @@ -736,8 +786,8 @@ if (rtc_include_tests) { "../../test:test_support", "../../test:video_test_common", "../../test:video_test_support", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } rtc_library("videocodec_test_stats_impl") { @@ -769,11 +819,17 @@ if (rtc_include_tests) { "codecs/vp8/test/vp8_impl_unittest.cc", "codecs/vp9/test/vp9_impl_unittest.cc", ] + + # TODO(jianj): Fix crash on iOS and re-enable + if (enable_libaom && !is_ios) { + sources += [ "codecs/test/videocodec_test_libaom.cc" ] + } if (rtc_use_h264) { sources += [ "codecs/test/videocodec_test_openh264.cc" ] } deps = [ + ":encoded_video_frame_producer", ":video_codec_interface", ":video_codecs_test_framework", ":video_coding_utility", @@ -812,9 +868,9 @@ if (rtc_include_tests) { "../../test:test_support", "../../test:video_test_common", "../rtp_rtcp:rtp_rtcp_format", - "//third_party/abseil-cpp/absl/types:optional", "//third_party/libyuv", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] data = video_coding_modules_tests_resources @@ -843,6 +899,7 @@ if (rtc_include_tests) { testonly = true sources = [ + "chain_diff_calculator_unittest.cc", "codecs/test/videocodec_test_fixture_config_unittest.cc", "codecs/test/videocodec_test_stats_impl_unittest.cc", "codecs/test/videoprocessor_unittest.cc", @@ -862,6 +919,7 @@ if (rtc_include_tests) { "jitter_buffer_unittest.cc", "jitter_estimator_tests.cc", "loss_notification_controller_unittest.cc", + "nack_module2_unittest.cc", "nack_module_unittest.cc", "packet_buffer_unittest.cc", "receiver_unittest.cc", @@ -889,6 +947,7 @@ if (rtc_include_tests) { } deps = [ + ":chain_diff_calculator", ":codec_globals_headers", ":encoded_frame", ":frame_dependencies_calculator", @@ -955,7 +1014,11 @@ if (rtc_include_tests) { "../../test/time_controller:time_controller", "../rtp_rtcp:rtp_rtcp_format", "../rtp_rtcp:rtp_video_header", + "codecs/av1:scalability_structure_tests", "codecs/av1:video_coding_codecs_av1_tests", + "deprecated:nack_module", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/types:optional", "//third_party/abseil-cpp/absl/types:variant", diff --git a/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator.cc b/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator.cc new file mode 100644 index 00000000000..5f852717b5d --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/chain_diff_calculator.h" + +#include <stddef.h> +#include <stdint.h> + +#include <algorithm> +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +void ChainDiffCalculator::Reset(const std::vector<bool>& chains) { + last_frame_in_chain_.resize(chains.size()); + for (size_t i = 0; i < chains.size(); ++i) { + if (chains[i]) { + last_frame_in_chain_[i] = absl::nullopt; + } + } +} + +absl::InlinedVector<int, 4> ChainDiffCalculator::ChainDiffs( + int64_t frame_id) const { + absl::InlinedVector<int, 4> result; + result.reserve(last_frame_in_chain_.size()); + for (const auto& frame_id_in_chain : last_frame_in_chain_) { + result.push_back(frame_id_in_chain ? (frame_id - *frame_id_in_chain) : 0); + } + return result; +} + +absl::InlinedVector<int, 4> ChainDiffCalculator::From( + int64_t frame_id, + const std::vector<bool>& chains) { + auto result = ChainDiffs(frame_id); + if (chains.size() != last_frame_in_chain_.size()) { + RTC_LOG(LS_ERROR) << "Insconsistent chain configuration for frame#" + << frame_id << ": expected " + << last_frame_in_chain_.size() << " chains, found " + << chains.size(); + } + size_t num_chains = std::min(last_frame_in_chain_.size(), chains.size()); + for (size_t i = 0; i < num_chains; ++i) { + if (chains[i]) { + last_frame_in_chain_[i] = frame_id; + } + } + return result; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator.h b/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator.h new file mode 100644 index 00000000000..bca7340c6f7 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CHAIN_DIFF_CALCULATOR_H_ +#define MODULES_VIDEO_CODING_CHAIN_DIFF_CALCULATOR_H_ + +#include <stdint.h> + +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" + +namespace webrtc { + +// This class is thread compatible. +class ChainDiffCalculator { + public: + ChainDiffCalculator() = default; + ChainDiffCalculator(const ChainDiffCalculator&) = default; + ChainDiffCalculator& operator=(const ChainDiffCalculator&) = default; + + // Restarts chains, i.e. for position where chains[i] == true next chain_diff + // will be 0. Saves chains.size() as number of chains in the stream. + void Reset(const std::vector<bool>& chains); + + // Returns chain diffs based on flags if frame is part of the chain. + absl::InlinedVector<int, 4> From(int64_t frame_id, + const std::vector<bool>& chains); + + private: + absl::InlinedVector<int, 4> ChainDiffs(int64_t frame_id) const; + + absl::InlinedVector<absl::optional<int64_t>, 4> last_frame_in_chain_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CHAIN_DIFF_CALCULATOR_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator_unittest.cc new file mode 100644 index 00000000000..efd09bd8880 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/chain_diff_calculator_unittest.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/chain_diff_calculator.h" + +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::ElementsAre; +using ::testing::SizeIs; + +TEST(ChainDiffCalculatorTest, SingleChain) { + // Simulate a stream with 2 temporal layer where chain + // protects temporal layer 0. + ChainDiffCalculator calculator; + // Key frame. + calculator.Reset({true}); + EXPECT_THAT(calculator.From(1, {true}), ElementsAre(0)); + // T1 delta frame. + EXPECT_THAT(calculator.From(2, {false}), ElementsAre(1)); + // T0 delta frame. + EXPECT_THAT(calculator.From(3, {true}), ElementsAre(2)); +} + +TEST(ChainDiffCalculatorTest, TwoChainsFullSvc) { + // Simulate a full svc stream with 2 spatial and 2 temporal layers. + // chains are protecting temporal layers 0. + ChainDiffCalculator calculator; + // S0 Key frame. + calculator.Reset({true, true}); + EXPECT_THAT(calculator.From(1, {true, true}), ElementsAre(0, 0)); + // S1 Key frame. + EXPECT_THAT(calculator.From(2, {false, true}), ElementsAre(1, 1)); + // S0T1 delta frame. + EXPECT_THAT(calculator.From(3, {false, false}), ElementsAre(2, 1)); + // S1T1 delta frame. + EXPECT_THAT(calculator.From(4, {false, false}), ElementsAre(3, 2)); + // S0T0 delta frame. + EXPECT_THAT(calculator.From(5, {true, true}), ElementsAre(4, 3)); + // S1T0 delta frame. + EXPECT_THAT(calculator.From(6, {false, true}), ElementsAre(1, 1)); +} + +TEST(ChainDiffCalculatorTest, TwoChainsKSvc) { + // Simulate a k-svc stream with 2 spatial and 2 temporal layers. + // chains are protecting temporal layers 0. + ChainDiffCalculator calculator; + // S0 Key frame. + calculator.Reset({true, true}); + EXPECT_THAT(calculator.From(1, {true, true}), ElementsAre(0, 0)); + // S1 Key frame. + EXPECT_THAT(calculator.From(2, {false, true}), ElementsAre(1, 1)); + // S0T1 delta frame. + EXPECT_THAT(calculator.From(3, {false, false}), ElementsAre(2, 1)); + // S1T1 delta frame. + EXPECT_THAT(calculator.From(4, {false, false}), ElementsAre(3, 2)); + // S0T0 delta frame. + EXPECT_THAT(calculator.From(5, {true, false}), ElementsAre(4, 3)); + // S1T0 delta frame. + EXPECT_THAT(calculator.From(6, {false, true}), ElementsAre(1, 4)); +} + +TEST(ChainDiffCalculatorTest, TwoChainsSimulcast) { + // Simulate a k-svc stream with 2 spatial and 2 temporal layers. + // chains are protecting temporal layers 0. + ChainDiffCalculator calculator; + // S0 Key frame. + calculator.Reset({true, false}); + EXPECT_THAT(calculator.From(1, {true, false}), ElementsAre(0, 0)); + // S1 Key frame. + calculator.Reset({false, true}); + EXPECT_THAT(calculator.From(2, {false, true}), ElementsAre(1, 0)); + // S0T1 delta frame. + EXPECT_THAT(calculator.From(3, {false, false}), ElementsAre(2, 1)); + // S1T1 delta frame. + EXPECT_THAT(calculator.From(4, {false, false}), ElementsAre(3, 2)); + // S0T0 delta frame. + EXPECT_THAT(calculator.From(5, {true, false}), ElementsAre(4, 3)); + // S1T0 delta frame. + EXPECT_THAT(calculator.From(6, {false, true}), ElementsAre(1, 4)); +} + +TEST(ChainDiffCalculatorTest, ResilentToAbsentChainConfig) { + ChainDiffCalculator calculator; + // Key frame. + calculator.Reset({true, false}); + EXPECT_THAT(calculator.From(1, {true, false}), ElementsAre(0, 0)); + // Forgot to set chains. should still return 2 chain_diffs. + EXPECT_THAT(calculator.From(2, {}), ElementsAre(1, 0)); + // chain diffs for next frame(s) are undefined, but still there should be + // correct number of them. + EXPECT_THAT(calculator.From(3, {true, false}), SizeIs(2)); + EXPECT_THAT(calculator.From(4, {false, true}), SizeIs(2)); + // Since previous two frames updated all the chains, can expect what + // chain_diffs would be. + EXPECT_THAT(calculator.From(5, {false, false}), ElementsAre(2, 1)); +} + +TEST(ChainDiffCalculatorTest, ResilentToTooMainChains) { + ChainDiffCalculator calculator; + // Key frame. + calculator.Reset({true, false}); + EXPECT_THAT(calculator.From(1, {true, false}), ElementsAre(0, 0)); + // Set wrong number of chains. Expect number of chain_diffs is not changed. + EXPECT_THAT(calculator.From(2, {true, true, true}), ElementsAre(1, 0)); + // chain diffs for next frame(s) are undefined, but still there should be + // correct number of them. + EXPECT_THAT(calculator.From(3, {true, false}), SizeIs(2)); + EXPECT_THAT(calculator.From(4, {false, true}), SizeIs(2)); + // Since previous two frames updated all the chains, can expect what + // chain_diffs would be. + EXPECT_THAT(calculator.From(5, {false, false}), ElementsAre(2, 1)); +} + +} // namespace +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/BUILD.gn b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/BUILD.gn index b2b82d49472..a927db293d4 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/BUILD.gn +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/BUILD.gn @@ -13,10 +13,8 @@ rtc_library("libaom_av1_decoder") { visibility = [ "*" ] poisonous = [ "software_video_codecs" ] public = [ "libaom_av1_decoder.h" ] - deps = [ - "../../../../api/video_codecs:video_codecs_api", - "//third_party/abseil-cpp/absl/base:core_headers", - ] + deps = [ "../../../../api/video_codecs:video_codecs_api" ] + absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ] if (enable_libaom) { sources = [ "libaom_av1_decoder.cc" ] @@ -27,21 +25,79 @@ rtc_library("libaom_av1_decoder") { "../../../../api/video:video_frame_i420", "../../../../common_video", "../../../../rtc_base:logging", - "//third_party/abseil-cpp/absl/types:optional", "//third_party/libaom", "//third_party/libyuv", ] + absl_deps += [ "//third_party/abseil-cpp/absl/types:optional" ] } else { sources = [ "libaom_av1_decoder_absent.cc" ] } } +rtc_source_set("scalable_video_controller") { + sources = [ + "scalable_video_controller.h", + "scalable_video_controller_no_layering.cc", + "scalable_video_controller_no_layering.h", + ] + deps = [ + "../../../../api/transport/rtp:dependency_descriptor", + "../../../../api/video:video_bitrate_allocation", + "../../../../common_video/generic_frame_descriptor", + "../../../../rtc_base:checks", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + +rtc_source_set("scalability_structures") { + sources = [ + "scalability_structure_l1t2.cc", + "scalability_structure_l1t2.h", + "scalability_structure_l1t3.cc", + "scalability_structure_l1t3.h", + "scalability_structure_l2t1.cc", + "scalability_structure_l2t1.h", + "scalability_structure_l2t1_key.cc", + "scalability_structure_l2t1_key.h", + "scalability_structure_l2t2.cc", + "scalability_structure_l2t2.h", + "scalability_structure_l2t2_key.cc", + "scalability_structure_l2t2_key.h", + "scalability_structure_l2t2_key_shift.cc", + "scalability_structure_l2t2_key_shift.h", + "scalability_structure_l3t1.cc", + "scalability_structure_l3t1.h", + "scalability_structure_l3t3.cc", + "scalability_structure_l3t3.h", + "scalability_structure_s2t1.cc", + "scalability_structure_s2t1.h", + ] + deps = [ + ":scalable_video_controller", + "../../../../api/transport/rtp:dependency_descriptor", + "../../../../common_video/generic_frame_descriptor", + "../../../../rtc_base:checks", + "../../../../rtc_base:logging", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + rtc_library("libaom_av1_encoder") { visibility = [ "*" ] poisonous = [ "software_video_codecs" ] public = [ "libaom_av1_encoder.h" ] deps = [ + ":scalable_video_controller", "../../../../api/video_codecs:video_codecs_api", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/base:core_headers", ] @@ -56,7 +112,6 @@ rtc_library("libaom_av1_encoder") { "../../../../common_video", "../../../../rtc_base:checks", "../../../../rtc_base:logging", - "//third_party/abseil-cpp/absl/algorithm:container", "//third_party/libaom", ] } else { @@ -65,6 +120,21 @@ rtc_library("libaom_av1_encoder") { } if (rtc_include_tests) { + rtc_library("scalability_structure_tests") { + testonly = true + sources = [ "scalability_structure_unittest.cc" ] + deps = [ + ":scalability_structures", + ":scalable_video_controller", + "../..:chain_diff_calculator", + "../..:frame_dependencies_calculator", + "../../../../api/transport/rtp:dependency_descriptor", + "../../../../api/video:video_frame_type", + "../../../../test:test_support", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] + } + rtc_library("video_coding_codecs_av1_tests") { testonly = true @@ -76,13 +146,16 @@ if (rtc_include_tests) { deps = [ ":libaom_av1_decoder", ":libaom_av1_encoder", + ":scalability_structures", + ":scalable_video_controller", + "../..:encoded_video_frame_producer", "../..:video_codec_interface", - "../../../../api:create_frame_generator", - "../../../../api:frame_generator_api", + "../../../../api:mock_video_encoder", + "../../../../api/video:video_frame_i420", "../../../../api/video_codecs:video_codecs_api", "../../../../test:test_support", - "//third_party/abseil-cpp/absl/types:optional", ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } } } diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_decoder.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_decoder.cc index 122f214a5c8..1a8a0c4775a 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_decoder.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_decoder.cc @@ -53,6 +53,8 @@ class LibaomAv1Decoder final : public VideoDecoder { int32_t Release() override; + const char* ImplementationName() const override; + private: aom_codec_ctx_t context_; bool inited_; @@ -127,7 +129,7 @@ int32_t LibaomAv1Decoder::Decode(const EncodedImage& encoded_image, // Return decoded frame data. int qp; - ret = aom_codec_control_(&context_, AOMD_GET_LAST_QUANTIZER, &qp); + ret = aom_codec_control(&context_, AOMD_GET_LAST_QUANTIZER, &qp); if (ret != AOM_CODEC_OK) { RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret << " on control AOME_GET_LAST_QUANTIZER."; @@ -180,6 +182,10 @@ int32_t LibaomAv1Decoder::Release() { return WEBRTC_VIDEO_CODEC_OK; } +const char* LibaomAv1Decoder::ImplementationName() const { + return "libaom"; +} + } // namespace const bool kIsLibaomAv1DecoderSupported = true; diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index 59ad127435e..0b2c2dacf79 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -13,15 +13,19 @@ #include <stdint.h> #include <memory> +#include <utility> #include <vector> #include "absl/algorithm/container.h" +#include "absl/base/macros.h" #include "api/scoped_refptr.h" #include "api/video/encoded_image.h" #include "api/video/i420_buffer.h" #include "api/video/video_frame.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" @@ -34,9 +38,7 @@ namespace webrtc { namespace { // Encoder configuration parameters -constexpr int kQpMax = 56; constexpr int kQpMin = 10; -constexpr int kDefaultEncSpeed = 7; // Use values 6, 7, or 8 for RTC. constexpr int kUsageProfile = 1; // 0 = good quality; 1 = real-time. constexpr int kMinQindex = 58; // Min qindex threshold for QP scaling. constexpr int kMaxQindex = 180; // Max qindex threshold for QP scaling. @@ -45,9 +47,23 @@ constexpr int kLagInFrames = 0; // No look ahead. constexpr int kRtpTicksPerSecond = 90000; constexpr float kMinimumFrameRate = 1.0; +// Only positive speeds, range for real-time coding currently is: 6 - 8. +// Lower means slower/better quality, higher means fastest/lower quality. +int GetCpuSpeed(int width, int height, int number_of_cores) { + // For smaller resolutions, use lower speed setting (get some coding gain at + // the cost of increased encoding complexity). + if (number_of_cores > 2 && width * height <= 320 * 180) + return 6; + else if (width * height >= 1280 * 720) + return 8; + else + return 7; +} + class LibaomAv1Encoder final : public VideoEncoder { public: - LibaomAv1Encoder(); + explicit LibaomAv1Encoder( + std::unique_ptr<ScalableVideoController> svc_controller); ~LibaomAv1Encoder(); int InitEncode(const VideoCodec* codec_settings, @@ -66,8 +82,18 @@ class LibaomAv1Encoder final : public VideoEncoder { EncoderInfo GetEncoderInfo() const override; private: + // Configures the encoder with scalability for the next coded video sequence. + bool SetSvcParams(ScalableVideoController::StreamLayersConfig svc_config); + // Configures the encoder with layer for the next frame. + void SetSvcLayerId( + const ScalableVideoController::LayerFrameConfig& layer_frame); + // Configures the encoder which buffers next frame updates and can reference. + void SetSvcRefFrameConfig( + const ScalableVideoController::LayerFrameConfig& layer_frame); + + const std::unique_ptr<ScalableVideoController> svc_controller_; bool inited_; - bool keyframe_required_; + bool svc_enabled_; VideoCodec encoder_settings_; aom_image_t* frame_for_encode_; aom_codec_ctx_t ctx_; @@ -100,11 +126,15 @@ int32_t VerifyCodecSettings(const VideoCodec& codec_settings) { return WEBRTC_VIDEO_CODEC_OK; } -LibaomAv1Encoder::LibaomAv1Encoder() - : inited_(false), - keyframe_required_(true), +LibaomAv1Encoder::LibaomAv1Encoder( + std::unique_ptr<ScalableVideoController> svc_controller) + : svc_controller_(std::move(svc_controller)), + inited_(false), + svc_enabled_(false), frame_for_encode_(nullptr), - encoded_image_callback_(nullptr) {} + encoded_image_callback_(nullptr) { + RTC_DCHECK(svc_controller_); +} LibaomAv1Encoder::~LibaomAv1Encoder() { Release(); @@ -153,9 +183,12 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, cfg_.g_input_bit_depth = kBitDepth; cfg_.kf_mode = AOM_KF_DISABLED; cfg_.rc_min_quantizer = kQpMin; - cfg_.rc_max_quantizer = kQpMax; + cfg_.rc_max_quantizer = encoder_settings_.qpMax; cfg_.g_usage = kUsageProfile; - + if (svc_controller_->StreamConfig().num_spatial_layers > 1 || + svc_controller_->StreamConfig().num_temporal_layers > 1) { + cfg_.g_error_resilient = 1; + } // Low-latency settings. cfg_.rc_end_usage = AOM_CBR; // Constant Bit Rate (CBR) mode cfg_.g_pass = AOM_RC_ONE_PASS; // One-pass rate control @@ -180,7 +213,9 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, inited_ = true; // Set control parameters - ret = aom_codec_control(&ctx_, AOME_SET_CPUUSED, kDefaultEncSpeed); + ret = aom_codec_control( + &ctx_, AOME_SET_CPUUSED, + GetCpuSpeed(cfg_.g_w, cfg_.g_h, settings.number_of_cores)); if (ret != AOM_CODEC_OK) { RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret << " on control AV1E_SET_CPUUSED."; @@ -198,16 +233,149 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, << " on control AV1E_SET_DELTAQ_MODE."; return WEBRTC_VIDEO_CODEC_ERROR; } + ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_ORDER_HINT, 0); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret + << " on control AV1E_SET_ENABLE_ORDER_HINT."; + return WEBRTC_VIDEO_CODEC_ERROR; + } ret = aom_codec_control(&ctx_, AV1E_SET_AQ_MODE, 3); if (ret != AOM_CODEC_OK) { RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret << " on control AV1E_SET_AQ_MODE."; return WEBRTC_VIDEO_CODEC_ERROR; } + if (!SetSvcParams(svc_controller_->StreamConfig())) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + ret = aom_codec_control(&ctx_, AOME_SET_MAX_INTRA_BITRATE_PCT, 300); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret + << " on control AV1E_SET_MAX_INTRA_BITRATE_PCT."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + ret = aom_codec_control(&ctx_, AV1E_SET_COEFF_COST_UPD_FREQ, 2); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret + << " on control AV1E_SET_COEFF_COST_UPD_FREQ."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + ret = aom_codec_control(&ctx_, AV1E_SET_MODE_COST_UPD_FREQ, 2); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret + << " on control AV1E_SET_MODE_COST_UPD_FREQ."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + ret = aom_codec_control(&ctx_, AV1E_SET_MV_COST_UPD_FREQ, 3); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret + << " on control AV1E_SET_MV_COST_UPD_FREQ."; + return WEBRTC_VIDEO_CODEC_ERROR; + } return WEBRTC_VIDEO_CODEC_OK; } +bool LibaomAv1Encoder::SetSvcParams( + ScalableVideoController::StreamLayersConfig svc_config) { + svc_enabled_ = + svc_config.num_spatial_layers > 1 || svc_config.num_temporal_layers > 1; + if (!svc_enabled_) { + return true; + } + aom_svc_params_t svc_params = {}; + if (svc_config.num_spatial_layers < 1 || svc_config.num_spatial_layers > 4) { + RTC_LOG(LS_WARNING) << "Av1 supports up to 4 spatial layers. " + << svc_config.num_spatial_layers << " configured."; + return false; + } + if (svc_config.num_temporal_layers < 1 || + svc_config.num_temporal_layers > 8) { + RTC_LOG(LS_WARNING) << "Av1 supports up to 8 temporal layers. " + << svc_config.num_temporal_layers << " configured."; + return false; + } + svc_params.number_spatial_layers = svc_config.num_spatial_layers; + svc_params.number_temporal_layers = svc_config.num_temporal_layers; + + int num_layers = + svc_config.num_spatial_layers * svc_config.num_temporal_layers; + for (int i = 0; i < num_layers; ++i) { + svc_params.min_quantizers[i] = kQpMin; + svc_params.max_quantizers[i] = encoder_settings_.qpMax; + } + + // Assume each temporal layer doubles framerate. + for (int tid = 0; tid < svc_config.num_temporal_layers; ++tid) { + svc_params.framerate_factor[tid] = + 1 << (svc_config.num_temporal_layers - tid - 1); + } + + // TODO(danilchap): Add support for custom resolution factor. + for (int sid = 0; sid < svc_config.num_spatial_layers; ++sid) { + svc_params.scaling_factor_num[sid] = 1; + svc_params.scaling_factor_den[sid] = + 1 << (svc_config.num_spatial_layers - sid - 1); + } + + aom_codec_err_t ret = + aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &svc_params); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAV1Encoder::EncodeInit returned " << ret + << " on control AV1E_SET_SVC_PARAMS."; + return false; + } + return true; +} + +void LibaomAv1Encoder::SetSvcLayerId( + const ScalableVideoController::LayerFrameConfig& layer_frame) { + aom_svc_layer_id_t layer_id = {}; + layer_id.spatial_layer_id = layer_frame.SpatialId(); + layer_id.temporal_layer_id = layer_frame.TemporalId(); + aom_codec_err_t ret = + aom_codec_control(&ctx_, AV1E_SET_SVC_LAYER_ID, &layer_id); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret + << " on control AV1E_SET_SVC_LAYER_ID."; + } +} + +void LibaomAv1Encoder::SetSvcRefFrameConfig( + const ScalableVideoController::LayerFrameConfig& layer_frame) { + // Buffer name to use for each layer_frame.buffers position. In particular + // when there are 2 buffers are referenced, prefer name them last and golden, + // because av1 bitstream format has dedicated fields for these two names. + // See last_frame_idx and golden_frame_idx in the av1 spec + // https://aomediacodec.github.io/av1-spec/av1-spec.pdf + static constexpr int kPreferedSlotName[] = {0, // Last + 3, // Golden + 1, 2, 4, 5, 6}; + static constexpr int kAv1NumBuffers = 8; + + aom_svc_ref_frame_config_t ref_frame_config = {}; + RTC_CHECK_LE(layer_frame.Buffers().size(), ABSL_ARRAYSIZE(kPreferedSlotName)); + for (size_t i = 0; i < layer_frame.Buffers().size(); ++i) { + const CodecBufferUsage& buffer = layer_frame.Buffers()[i]; + int slot_name = kPreferedSlotName[i]; + RTC_CHECK_GE(buffer.id, 0); + RTC_CHECK_LT(buffer.id, kAv1NumBuffers); + ref_frame_config.ref_idx[slot_name] = buffer.id; + if (buffer.referenced) { + ref_frame_config.reference[slot_name] = 1; + } + if (buffer.updated) { + ref_frame_config.refresh[buffer.id] = 1; + } + } + aom_codec_err_t ret = aom_codec_control(&ctx_, AV1E_SET_SVC_REF_FRAME_CONFIG, + &ref_frame_config); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret + << " on control AV1_SET_SVC_REF_FRAME_CONFIG."; + } +} + int32_t LibaomAv1Encoder::RegisterEncodeCompleteCallback( EncodedImageCallback* encoded_image_callback) { encoded_image_callback_ = encoded_image_callback; @@ -235,10 +403,18 @@ int32_t LibaomAv1Encoder::Encode( return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } - keyframe_required_ = + bool keyframe_required = frame_types != nullptr && absl::c_linear_search(*frame_types, VideoFrameType::kVideoFrameKey); + std::vector<ScalableVideoController::LayerFrameConfig> layer_frames = + svc_controller_->NextFrameConfig(keyframe_required); + + if (layer_frames.empty()) { + RTC_LOG(LS_ERROR) << "SVCController returned no configuration for a frame."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + // Convert input frame to I420, if needed. VideoFrame prepped_input_frame = frame; if (prepped_input_frame.video_frame_buffer()->type() != @@ -263,75 +439,84 @@ int32_t LibaomAv1Encoder::Encode( const uint32_t duration = kRtpTicksPerSecond / static_cast<float>(encoder_settings_.maxFramerate); - aom_enc_frame_flags_t flags = (keyframe_required_) ? AOM_EFLAG_FORCE_KF : 0; - // Encode a frame. - aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_, - frame.timestamp(), duration, flags); - if (ret != AOM_CODEC_OK) { - RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret - << " on aom_codec_encode."; - return WEBRTC_VIDEO_CODEC_ERROR; - } + for (ScalableVideoController::LayerFrameConfig& layer_frame : layer_frames) { + aom_enc_frame_flags_t flags = + layer_frame.IsKeyframe() ? AOM_EFLAG_FORCE_KF : 0; - // Get encoded image data. - EncodedImage encoded_image; - encoded_image._completeFrame = true; - aom_codec_iter_t iter = nullptr; - int data_pkt_count = 0; - while (const aom_codec_cx_pkt_t* pkt = aom_codec_get_cx_data(&ctx_, &iter)) { - if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) { - if (data_pkt_count > 0) { - RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than " - "one data packet for an input video frame."; - Release(); - } - // TODO(bugs.webrtc.org/11174): Remove this hack when - // webrtc_pc_e2e::SingleProcessEncodedImageDataInjector not used or fixed - // not to assume that encoded image transfered as is. - const uint8_t* data = static_cast<const uint8_t*>(pkt->data.frame.buf); - size_t size = pkt->data.frame.sz; - if (size > 2 && data[0] == 0b0'0010'010 && data[1] == 0) { - // Typically frame starts with a Temporal Delimter OBU of size 0 that is - // not need by any component in webrtc and discarded during rtp - // packetization. Before discarded it confuses test framework that - // assumes received encoded frame is exactly same as sent frame. - data += 2; - size -= 2; - } - encoded_image.SetEncodedData(EncodedImageBuffer::Create(data, size)); - - bool is_key_frame = ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0); - encoded_image._frameType = is_key_frame - ? VideoFrameType::kVideoFrameKey - : VideoFrameType::kVideoFrameDelta; - encoded_image.SetTimestamp(frame.timestamp()); - encoded_image.capture_time_ms_ = frame.render_time_ms(); - encoded_image.rotation_ = frame.rotation(); - encoded_image.content_type_ = VideoContentType::UNSPECIFIED; - // If encoded image width/height info are added to aom_codec_cx_pkt_t, - // use those values in lieu of the values in frame. - encoded_image._encodedHeight = frame.height(); - encoded_image._encodedWidth = frame.width(); - encoded_image.timing_.flags = VideoSendTiming::kInvalid; - int qp = -1; - ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp); - if (ret != AOM_CODEC_OK) { - RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret - << " on control AOME_GET_LAST_QUANTIZER."; - return WEBRTC_VIDEO_CODEC_ERROR; + if (svc_enabled_) { + SetSvcLayerId(layer_frame); + SetSvcRefFrameConfig(layer_frame); + } + + // Encode a frame. + aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_, + frame.timestamp(), duration, flags); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret + << " on aom_codec_encode."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Get encoded image data. + EncodedImage encoded_image; + encoded_image._completeFrame = true; + aom_codec_iter_t iter = nullptr; + int data_pkt_count = 0; + while (const aom_codec_cx_pkt_t* pkt = + aom_codec_get_cx_data(&ctx_, &iter)) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) { + if (data_pkt_count > 0) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than " + "one data packet for an input video frame."; + Release(); + } + encoded_image.SetEncodedData(EncodedImageBuffer::Create( + /*data=*/static_cast<const uint8_t*>(pkt->data.frame.buf), + /*size=*/pkt->data.frame.sz)); + + if ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0) { + layer_frame.Keyframe(); + } + encoded_image._frameType = layer_frame.IsKeyframe() + ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + encoded_image.SetTimestamp(frame.timestamp()); + encoded_image.capture_time_ms_ = frame.render_time_ms(); + encoded_image.rotation_ = frame.rotation(); + encoded_image.content_type_ = VideoContentType::UNSPECIFIED; + // If encoded image width/height info are added to aom_codec_cx_pkt_t, + // use those values in lieu of the values in frame. + encoded_image._encodedHeight = frame.height(); + encoded_image._encodedWidth = frame.width(); + encoded_image.timing_.flags = VideoSendTiming::kInvalid; + int qp = -1; + ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret + << " on control AOME_GET_LAST_QUANTIZER."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + encoded_image.qp_ = qp; + encoded_image.SetColorSpace(frame.color_space()); + ++data_pkt_count; } - encoded_image.qp_ = qp; - encoded_image.SetColorSpace(frame.color_space()); - ++data_pkt_count; } - } - // Deliver encoded image data. - if (encoded_image.size() > 0) { - CodecSpecificInfo codec_specific_info; - encoded_image_callback_->OnEncodedImage(encoded_image, &codec_specific_info, - nullptr); + // Deliver encoded image data. + if (encoded_image.size() > 0) { + CodecSpecificInfo codec_specific_info; + codec_specific_info.codecType = kVideoCodecAV1; + bool is_keyframe = layer_frame.IsKeyframe(); + codec_specific_info.generic_frame_info = + svc_controller_->OnEncodeDone(std::move(layer_frame)); + if (is_keyframe && codec_specific_info.generic_frame_info) { + codec_specific_info.template_structure = + svc_controller_->DependencyStructure(); + } + encoded_image_callback_->OnEncodedImage(encoded_image, + &codec_specific_info, nullptr); + } } return WEBRTC_VIDEO_CODEC_OK; @@ -359,6 +544,7 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) { RTC_DCHECK_LE(rc_target_bitrate_kbps, encoder_settings_.maxBitrate); RTC_DCHECK_GE(rc_target_bitrate_kbps, encoder_settings_.minBitrate); + svc_controller_->OnRatesUpdated(parameters.bitrate); // Set target bit rate. cfg_.rc_target_bitrate = rc_target_bitrate_kbps; @@ -389,7 +575,13 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const { const bool kIsLibaomAv1EncoderSupported = true; std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder() { - return std::make_unique<LibaomAv1Encoder>(); + return std::make_unique<LibaomAv1Encoder>( + std::make_unique<ScalableVideoControllerNoLayering>()); +} + +std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder( + std::unique_ptr<ScalableVideoController> svc_controller) { + return std::make_unique<LibaomAv1Encoder>(std::move(svc_controller)); } } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.h index 4b0ee28d402..c2f04e669cb 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder.h @@ -14,12 +14,15 @@ #include "absl/base/attributes.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" namespace webrtc { ABSL_CONST_INIT extern const bool kIsLibaomAv1EncoderSupported; std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder(); +std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder( + std::unique_ptr<ScalableVideoController> controller); } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc index 6d1d0bbb240..341a82774d5 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc @@ -11,15 +11,38 @@ #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" #include <memory> +#include <vector> +#include "absl/types/optional.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" #include "modules/video_coding/include/video_error_codes.h" +#include "test/gmock.h" #include "test/gtest.h" namespace webrtc { namespace { +using ::testing::SizeIs; + +VideoCodec DefaultCodecSettings() { + VideoCodec codec_settings; + codec_settings.width = 320; + codec_settings.height = 180; + codec_settings.maxFramerate = 30; + codec_settings.maxBitrate = 1000; + codec_settings.qpMax = 63; + return codec_settings; +} + +VideoEncoder::Settings DefaultEncoderSettings() { + return VideoEncoder::Settings( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, /*max_payload_size=*/1200); +} + TEST(LibaomAv1EncoderTest, CanCreate) { std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(); EXPECT_TRUE(encoder); @@ -28,17 +51,37 @@ TEST(LibaomAv1EncoderTest, CanCreate) { TEST(LibaomAv1EncoderTest, InitAndRelease) { std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(); ASSERT_TRUE(encoder); - VideoCodec codec_settings; - codec_settings.width = 1280; - codec_settings.height = 720; - codec_settings.maxFramerate = 30; - VideoEncoder::Capabilities capabilities(/*loss_notification=*/false); - VideoEncoder::Settings encoder_settings(capabilities, /*number_of_cores=*/1, - /*max_payload_size=*/1200); - EXPECT_EQ(encoder->InitEncode(&codec_settings, encoder_settings), + VideoCodec codec_settings = DefaultCodecSettings(); + EXPECT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), WEBRTC_VIDEO_CODEC_OK); EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK); } +TEST(LibaomAv1EncoderTest, NoBitrateOnTopLayerRefecltedInActiveDecodeTargets) { + // Configure encoder with 2 temporal layers. + std::unique_ptr<VideoEncoder> encoder = + CreateLibaomAv1Encoder(std::make_unique<ScalabilityStructureL1T2>()); + VideoCodec codec_settings = DefaultCodecSettings(); + ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), + WEBRTC_VIDEO_CODEC_OK); + + VideoEncoder::RateControlParameters rate_parameters; + rate_parameters.framerate_fps = 30; + rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/0, 300'000); + rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/1, 0); + encoder->SetRates(rate_parameters); + + std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames = + EncodedVideoFrameProducer(*encoder).SetNumInputFrames(1).Encode(); + ASSERT_THAT(encoded_frames, SizeIs(1)); + ASSERT_NE(encoded_frames[0].codec_specific_info.generic_frame_info, + absl::nullopt); + // Assuming L1T2 structure uses 1st decode target for T0 and 2nd decode target + // for T0+T1 frames, expect only 1st decode target is active. + EXPECT_EQ(encoded_frames[0] + .codec_specific_info.generic_frame_info->active_decode_targets, + 0b01); +} + } // namespace } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc index 4a549ea453d..c47a3923843 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/libaom_av1_unittest.cc @@ -15,12 +15,23 @@ #include <vector> #include "absl/types/optional.h" -#include "api/test/create_frame_generator.h" -#include "api/test/frame_generator_interface.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h" +#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h" +#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "test/gmock.h" @@ -29,79 +40,43 @@ namespace webrtc { namespace { +using ::testing::ContainerEq; +using ::testing::Each; using ::testing::ElementsAreArray; +using ::testing::Ge; using ::testing::IsEmpty; using ::testing::Not; using ::testing::NotNull; +using ::testing::SizeIs; +using ::testing::Truly; +using ::testing::Values; // Use small resolution for this test to make it faster. constexpr int kWidth = 320; constexpr int kHeight = 180; constexpr int kFramerate = 30; -constexpr int kRtpTicksPerSecond = 90000; -class TestAv1Encoder { - public: - struct Encoded { - EncodedImage encoded_image; - CodecSpecificInfo codec_specific_info; - }; - - TestAv1Encoder() : encoder_(CreateLibaomAv1Encoder()) { - RTC_CHECK(encoder_); - VideoCodec codec_settings; - codec_settings.width = kWidth; - codec_settings.height = kHeight; - codec_settings.maxFramerate = kFramerate; - VideoEncoder::Settings encoder_settings( - VideoEncoder::Capabilities(/*loss_notification=*/false), - /*number_of_cores=*/1, /*max_payload_size=*/1200); - EXPECT_EQ(encoder_->InitEncode(&codec_settings, encoder_settings), - WEBRTC_VIDEO_CODEC_OK); - EXPECT_EQ(encoder_->RegisterEncodeCompleteCallback(&callback_), - WEBRTC_VIDEO_CODEC_OK); - } - // This class requires pointer stability and thus not copyable nor movable. - TestAv1Encoder(const TestAv1Encoder&) = delete; - TestAv1Encoder& operator=(const TestAv1Encoder&) = delete; - - void EncodeAndAppend(const VideoFrame& frame, std::vector<Encoded>* encoded) { - callback_.SetEncodeStorage(encoded); - std::vector<VideoFrameType> frame_types = { - VideoFrameType::kVideoFrameDelta}; - EXPECT_EQ(encoder_->Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_OK); - // Prefer to crash checking nullptr rather than writing to random memory. - callback_.SetEncodeStorage(nullptr); - } - - private: - class EncoderCallback : public EncodedImageCallback { - public: - void SetEncodeStorage(std::vector<Encoded>* storage) { storage_ = storage; } - - private: - Result OnEncodedImage( - const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info, - const RTPFragmentationHeader* /*fragmentation*/) override { - RTC_CHECK(storage_); - storage_->push_back({encoded_image, *codec_specific_info}); - return Result(Result::Error::OK); - } - - std::vector<Encoded>* storage_ = nullptr; - }; - - EncoderCallback callback_; - std::unique_ptr<VideoEncoder> encoder_; -}; +VideoCodec DefaultCodecSettings() { + VideoCodec codec_settings; + codec_settings.width = kWidth; + codec_settings.height = kHeight; + codec_settings.maxFramerate = kFramerate; + codec_settings.maxBitrate = 1000; + codec_settings.qpMax = 63; + return codec_settings; +} +VideoEncoder::Settings DefaultEncoderSettings() { + return VideoEncoder::Settings( + VideoEncoder::Capabilities(/*loss_notification=*/false), + /*number_of_cores=*/1, /*max_payload_size=*/1200); +} class TestAv1Decoder { public: - TestAv1Decoder() { - decoder_ = CreateLibaomAv1Decoder(); + explicit TestAv1Decoder(int decoder_id) + : decoder_id_(decoder_id), decoder_(CreateLibaomAv1Decoder()) { if (decoder_ == nullptr) { - ADD_FAILURE() << "Failed to create a decoder"; + ADD_FAILURE() << "Failed to create a decoder#" << decoder_id_; return; } EXPECT_EQ(decoder_->InitDecode(/*codec_settings=*/nullptr, @@ -116,20 +91,17 @@ class TestAv1Decoder { void Decode(int64_t frame_id, const EncodedImage& image) { ASSERT_THAT(decoder_, NotNull()); - requested_ids_.push_back(frame_id); int32_t error = decoder_->Decode(image, /*missing_frames=*/false, /*render_time_ms=*/image.capture_time_ms_); if (error != WEBRTC_VIDEO_CODEC_OK) { ADD_FAILURE() << "Failed to decode frame id " << frame_id - << " with error code " << error; + << " with error code " << error << " by decoder#" + << decoder_id_; return; } decoded_ids_.push_back(frame_id); } - const std::vector<int64_t>& requested_frame_ids() const { - return requested_ids_; - } const std::vector<int64_t>& decoded_frame_ids() const { return decoded_ids_; } size_t num_output_frames() const { return callback_.num_called(); } @@ -156,51 +128,116 @@ class TestAv1Decoder { int num_called_ = 0; }; - std::vector<int64_t> requested_ids_; + const int decoder_id_; std::vector<int64_t> decoded_ids_; DecoderCallback callback_; - std::unique_ptr<VideoDecoder> decoder_; + const std::unique_ptr<VideoDecoder> decoder_; }; -std::vector<VideoFrame> GenerateFrames(size_t num_frames) { - std::vector<VideoFrame> frames; - frames.reserve(num_frames); - - auto input_frame_generator = test::CreateSquareFrameGenerator( - kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420, - absl::nullopt); - uint32_t timestamp = 1000; - for (size_t i = 0; i < num_frames; ++i) { - frames.push_back( - VideoFrame::Builder() - .set_video_frame_buffer(input_frame_generator->NextFrame().buffer) - .set_timestamp_rtp(timestamp += kRtpTicksPerSecond / kFramerate) - .build()); - } - return frames; -} - TEST(LibaomAv1Test, EncodeDecode) { - TestAv1Decoder decoder; - TestAv1Encoder encoder; - - std::vector<TestAv1Encoder::Encoded> encoded_frames; - for (const VideoFrame& frame : GenerateFrames(/*num_frames=*/4)) { - encoder.EncodeAndAppend(frame, &encoded_frames); - } - for (size_t frame_idx = 0; frame_idx < encoded_frames.size(); ++frame_idx) { - decoder.Decode(static_cast<int64_t>(frame_idx), - encoded_frames[frame_idx].encoded_image); + TestAv1Decoder decoder(0); + std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder(); + VideoCodec codec_settings = DefaultCodecSettings(); + ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), + WEBRTC_VIDEO_CODEC_OK); + + std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames = + EncodedVideoFrameProducer(*encoder).SetNumInputFrames(4).Encode(); + for (size_t frame_id = 0; frame_id < encoded_frames.size(); ++frame_id) { + decoder.Decode(static_cast<int64_t>(frame_id), + encoded_frames[frame_id].encoded_image); } // Check encoder produced some frames for decoder to decode. ASSERT_THAT(encoded_frames, Not(IsEmpty())); // Check decoder found all of them valid. - EXPECT_THAT(decoder.decoded_frame_ids(), - ElementsAreArray(decoder.requested_frame_ids())); + EXPECT_THAT(decoder.decoded_frame_ids(), SizeIs(encoded_frames.size())); // Check each of them produced an output frame. EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size()); } +struct SvcTestParam { + std::function<std::unique_ptr<ScalableVideoController>()> svc_factory; + int num_frames_to_generate; +}; + +class LibaomAv1SvcTest : public ::testing::TestWithParam<SvcTestParam> {}; + +TEST_P(LibaomAv1SvcTest, EncodeAndDecodeAllDecodeTargets) { + std::unique_ptr<ScalableVideoController> svc_controller = + GetParam().svc_factory(); + size_t num_decode_targets = + svc_controller->DependencyStructure().num_decode_targets; + + std::unique_ptr<VideoEncoder> encoder = + CreateLibaomAv1Encoder(std::move(svc_controller)); + VideoCodec codec_settings = DefaultCodecSettings(); + ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()), + WEBRTC_VIDEO_CODEC_OK); + std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames = + EncodedVideoFrameProducer(*encoder) + .SetNumInputFrames(GetParam().num_frames_to_generate) + .SetResolution({kWidth, kHeight}) + .Encode(); + + ASSERT_THAT( + encoded_frames, + Each(Truly([&](const EncodedVideoFrameProducer::EncodedFrame& frame) { + return frame.codec_specific_info.generic_frame_info && + frame.codec_specific_info.generic_frame_info + ->decode_target_indications.size() == num_decode_targets; + }))); + + for (size_t dt = 0; dt < num_decode_targets; ++dt) { + TestAv1Decoder decoder(dt); + std::vector<int64_t> requested_ids; + for (int64_t frame_id = 0; + frame_id < static_cast<int64_t>(encoded_frames.size()); ++frame_id) { + const EncodedVideoFrameProducer::EncodedFrame& frame = + encoded_frames[frame_id]; + if (frame.codec_specific_info.generic_frame_info + ->decode_target_indications[dt] != + DecodeTargetIndication::kNotPresent) { + requested_ids.push_back(frame_id); + decoder.Decode(frame_id, frame.encoded_image); + } + } + + ASSERT_THAT(requested_ids, SizeIs(Ge(2u))); + // Check decoder found all of them valid. + EXPECT_THAT(decoder.decoded_frame_ids(), ContainerEq(requested_ids)) + << "Decoder#" << dt; + // Check each of them produced an output frame. + EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size()) + << "Decoder#" << dt; + } +} + +INSTANTIATE_TEST_SUITE_P( + Svc, + LibaomAv1SvcTest, + Values(SvcTestParam{std::make_unique<ScalableVideoControllerNoLayering>, + /*num_frames_to_generate=*/4}, + SvcTestParam{std::make_unique<ScalabilityStructureL1T2>, + /*num_frames_to_generate=*/4}, + SvcTestParam{std::make_unique<ScalabilityStructureL1T3>, + /*num_frames_to_generate=*/8}, + SvcTestParam{std::make_unique<ScalabilityStructureL2T1>, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique<ScalabilityStructureL2T1Key>, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique<ScalabilityStructureL3T1>, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique<ScalabilityStructureL3T3>, + /*num_frames_to_generate=*/8}, + SvcTestParam{std::make_unique<ScalabilityStructureS2T1>, + /*num_frames_to_generate=*/3}, + SvcTestParam{std::make_unique<ScalabilityStructureL2T2>, + /*num_frames_to_generate=*/4}, + SvcTestParam{std::make_unique<ScalabilityStructureL2T2Key>, + /*num_frames_to_generate=*/4}, + SvcTestParam{std::make_unique<ScalabilityStructureL2T2KeyShift>, + /*num_frames_to_generate=*/4})); + } // namespace } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc new file mode 100644 index 00000000000..ae4c8792244 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t2.cc @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[3][2] = { + {kSwitch, kSwitch}, // KeyFrame + {kNotPresent, kDiscardable}, // DeltaFrame T1 + {kSwitch, kSwitch}, // DeltaFrame T0 +}; + +} // namespace + +ScalabilityStructureL1T2::~ScalabilityStructureL1T2() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL1T2::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 1; + result.num_temporal_layers = 2; + return result; +} + +FrameDependencyStructure ScalabilityStructureL1T2::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 1; + structure.decode_target_protected_by_chain = {0, 0}; + structure.templates.resize(3); + structure.templates[0].T(0).Dtis("SS").ChainDiffs({0}); + structure.templates[1].T(0).Dtis("SS").ChainDiffs({2}).FrameDiffs({2}); + structure.templates[2].T(1).Dtis("-D").ChainDiffs({1}).FrameDiffs({1}); + return structure; +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL1T2::NextFrameConfig(bool restart) { + if (!active_decode_targets_[0]) { + RTC_LOG(LS_WARNING) << "No bitrate allocated for temporal layer 0, yet " + "frame is requested. No frame will be encoded."; + return {}; + } + if (restart) { + next_pattern_ = kKeyFrame; + } else if (!active_decode_targets_[1]) { + next_pattern_ = kDeltaFrameT0; + } + std::vector<LayerFrameConfig> result(1); + + switch (next_pattern_) { + case kKeyFrame: + result[0].Id(0).T(0).Keyframe().Update(0); + next_pattern_ = kDeltaFrameT1; + break; + case kDeltaFrameT1: + result[0].Id(1).T(1).Reference(0); + next_pattern_ = kDeltaFrameT0; + break; + case kDeltaFrameT0: + result[0].Id(2).T(0).ReferenceAndUpdate(0); + next_pattern_ = kDeltaFrameT1; + break; + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL1T2::OnEncodeDone( + LayerFrameConfig config) { + // Encoder may have generated a keyframe even when not asked for it. Treat + // such frame same as requested keyframe, in particular restart the sequence. + if (config.IsKeyframe()) { + config = NextFrameConfig(/*restart=*/true).front(); + } + + absl::optional<GenericFrameInfo> frame_info; + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + frame_info->part_of_chain = {config.TemporalId() == 0}; + frame_info->active_decode_targets = active_decode_targets_; + return frame_info; +} + +void ScalabilityStructureL1T2::OnRatesUpdated( + const VideoBitrateAllocation& bitrates) { + if (bitrates.GetBitrate(0, 0) == 0) { + // It is unclear what frame can be produced when base layer is disabled, + // so mark all decode targets as inactive to produce no frames. + active_decode_targets_.reset(); + return; + } + active_decode_targets_.set(0, true); + active_decode_targets_.set(1, bitrates.GetBitrate(0, 1) > 0); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t2.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t2.h new file mode 100644 index 00000000000..55a9e8bbb0e --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t2.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_ + +#include <bitset> +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +class ScalabilityStructureL1T2 : public ScalableVideoController { + public: + ~ScalabilityStructureL1T2() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + void OnRatesUpdated(const VideoBitrateAllocation& bitrates) override; + + private: + enum FramePattern { + kKeyFrame, + kDeltaFrameT1, + kDeltaFrameT0, + }; + + FramePattern next_pattern_ = kKeyFrame; + std::bitset<32> active_decode_targets_ = 0b11; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t3.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t3.cc new file mode 100644 index 00000000000..a04a4262ed3 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t3.cc @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[3][3] = { + {kSwitch, kSwitch, kSwitch}, // T0 + {kNotPresent, kDiscardable, kSwitch}, // T1 + {kNotPresent, kNotPresent, kDiscardable}, // T2 +}; + +} // namespace + +ScalabilityStructureL1T3::~ScalabilityStructureL1T3() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL1T3::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 1; + result.num_temporal_layers = 3; + return result; +} + +FrameDependencyStructure ScalabilityStructureL1T3::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 3; + structure.num_chains = 1; + structure.decode_target_protected_by_chain = {0, 0, 0}; + structure.templates.resize(5); + structure.templates[0].T(0).Dtis("SSS").ChainDiffs({0}); + structure.templates[1].T(0).Dtis("SSS").ChainDiffs({4}).FrameDiffs({4}); + structure.templates[2].T(1).Dtis("-DS").ChainDiffs({2}).FrameDiffs({2}); + structure.templates[3].T(2).Dtis("--D").ChainDiffs({1}).FrameDiffs({1}); + structure.templates[4].T(2).Dtis("--D").ChainDiffs({3}).FrameDiffs({1}); + return structure; +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL1T3::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKeyFrame; + } + std::vector<LayerFrameConfig> config(1); + + switch (next_pattern_) { + case kKeyFrame: + config[0].T(0).Keyframe().Update(0); + next_pattern_ = kDeltaFrameT2A; + break; + case kDeltaFrameT2A: + config[0].T(2).Reference(0); + next_pattern_ = kDeltaFrameT1; + break; + case kDeltaFrameT1: + config[0].T(1).Reference(0).Update(1); + next_pattern_ = kDeltaFrameT2B; + break; + case kDeltaFrameT2B: + config[0].T(2).Reference(1); + next_pattern_ = kDeltaFrameT0; + break; + case kDeltaFrameT0: + config[0].T(0).ReferenceAndUpdate(0); + next_pattern_ = kDeltaFrameT2A; + break; + } + return config; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL1T3::OnEncodeDone( + LayerFrameConfig config) { + absl::optional<GenericFrameInfo> frame_info; + if (config.TemporalId() < 0 || + config.TemporalId() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected temporal id " << config.TemporalId(); + return frame_info; + } + frame_info.emplace(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign( + std::begin(kDtis[config.TemporalId()]), + std::end(kDtis[config.TemporalId()])); + frame_info->part_of_chain = {config.TemporalId() == 0}; + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t3.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t3.h new file mode 100644 index 00000000000..562d0f2a504 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l1t3.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_ + +#include <vector> + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// T2 0 0 0 0 +// | / | / +// T1 / 0 / 0 ... +// |_/ |_/ +// T0 0-------0------ +// Time-> 0 1 2 3 4 5 6 7 +class ScalabilityStructureL1T3 : public ScalableVideoController { + public: + ~ScalabilityStructureL1T3() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKeyFrame, + kDeltaFrameT2A, + kDeltaFrameT1, + kDeltaFrameT2B, + kDeltaFrameT0, + }; + + FramePattern next_pattern_ = kKeyFrame; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1.cc new file mode 100644 index 00000000000..c3cee19a294 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1.cc @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +constexpr DecodeTargetIndication kDtis[4][2] = { + {kSwitch, kSwitch}, // Key, S0 + {kNotPresent, kSwitch}, // Key, S1 + {kSwitch, kRequired}, // Delta, S0 + {kNotPresent, kRequired}, // Delta, S1 +}; + +} // namespace + +ScalabilityStructureL2T1::~ScalabilityStructureL2T1() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T1::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T1::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 1}; + structure.templates.resize(4); + structure.templates[0].S(0).Dtis("SR").ChainDiffs({2, 1}).FrameDiffs({2}); + structure.templates[1].S(0).Dtis("SS").ChainDiffs({0, 0}); + structure.templates[2].S(1).Dtis("-R").ChainDiffs({1, 1}).FrameDiffs({2, 1}); + structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 1}).FrameDiffs({1}); + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T1::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).S(0).Keyframe().Update(0); +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL2T1::NextFrameConfig(bool restart) { + std::vector<LayerFrameConfig> result(2); + // Buffer0 keeps latest S0 frame, Buffer1 keeps latest S1 frame. + if (restart || keyframe_) { + result[0] = KeyFrameConfig(); + result[1].Id(1).S(1).Reference(0).Update(1); + keyframe_ = false; + } else { + result[0].Id(2).S(0).ReferenceAndUpdate(0); + result[1].Id(3).S(1).Reference(0).ReferenceAndUpdate(1); + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL2T1::OnEncodeDone( + LayerFrameConfig config) { + absl::optional<GenericFrameInfo> frame_info; + if (config.IsKeyframe()) { + config = KeyFrameConfig(); + } + + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = std::move(config.Buffers()); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + frame_info->part_of_chain = {config.SpatialId() == 0, true}; + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1.h new file mode 100644 index 00000000000..0f536026046 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1 0--0--0- +// | | | ... +// S0 0--0--0- +class ScalabilityStructureL2T1 : public ScalableVideoController { + public: + ~ScalabilityStructureL2T1() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + LayerFrameConfig KeyFrameConfig() const; + + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc new file mode 100644 index 00000000000..7e273d1fc62 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[3][2] = { + {kSwitch, kSwitch}, // Key, S0 + {kSwitch, kNotPresent}, // Delta, S0 + {kNotPresent, kSwitch}, // Key and Delta, S1 +}; + +} // namespace + +ScalabilityStructureL2T1Key::~ScalabilityStructureL2T1Key() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T1Key::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T1Key::DependencyStructure() + const { + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 1}; + structure.templates.resize(4); + structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2}); + structure.templates[1].S(0).Dtis("SS").ChainDiffs({0, 0}); + structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2}); + structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 1}).FrameDiffs({1}); + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T1Key::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).S(0).Keyframe().Update(0); +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL2T1Key::NextFrameConfig(bool restart) { + std::vector<LayerFrameConfig> result(2); + + // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame. + if (restart || keyframe_) { + result[0] = KeyFrameConfig(); + result[1].Id(2).S(1).Reference(0).Update(1); + keyframe_ = false; + } else { + result[0].Id(1).S(0).ReferenceAndUpdate(0); + result[1].Id(2).S(1).ReferenceAndUpdate(1); + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL2T1Key::OnEncodeDone( + LayerFrameConfig config) { + absl::optional<GenericFrameInfo> frame_info; + if (config.IsKeyframe()) { + config = KeyFrameConfig(); + } + + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = std::move(config.Buffers()); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + if (config.IsKeyframe()) { + frame_info->part_of_chain = {true, true}; + } else { + frame_info->part_of_chain = {config.SpatialId() == 0, + config.SpatialId() == 1}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h new file mode 100644 index 00000000000..c1d8c8947fb --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1 0--0--0- +// | ... +// S0 0--0--0- +class ScalabilityStructureL2T1Key : public ScalableVideoController { + public: + ~ScalabilityStructureL2T1Key() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + LayerFrameConfig KeyFrameConfig() const; + + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2.cc new file mode 100644 index 00000000000..5db2fadb5f5 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2.cc @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +// decode targets: S0T0, S0T1, S1T0, S1T1 +constexpr DecodeTargetIndication kDtis[6][4] = { + {kSwitch, kSwitch, kSwitch, kSwitch}, // kKey, S0 + {kNotPresent, kNotPresent, kSwitch, kSwitch}, // kKey, S1 + {kNotPresent, kDiscardable, kNotPresent, kRequired}, // kDeltaT1, S0 + {kNotPresent, kNotPresent, kNotPresent, kDiscardable}, // kDeltaT1, S1 + {kSwitch, kSwitch, kRequired, kRequired}, // kDeltaT0, S0 + {kNotPresent, kNotPresent, kSwitch, kSwitch}, // kDeltaT0, S1 +}; + +} // namespace + +ScalabilityStructureL2T2::~ScalabilityStructureL2T2() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T2::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 2; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T2::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 4; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 0, 1, 1}; + structure.templates.resize(6); + auto& templates = structure.templates; + templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0}); + templates[1].S(0).T(0).Dtis("SSRR").ChainDiffs({4, 3}).FrameDiffs({4}); + templates[2].S(0).T(1).Dtis("-D-R").ChainDiffs({2, 1}).FrameDiffs({2}); + templates[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1}); + templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({4, 1}); + templates[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2, 1}); + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T2::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).Keyframe().S(0).T(0).Update(0); +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL2T2::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKey; + } + std::vector<LayerFrameConfig> result(2); + + // Buffer0 keeps latest S0T0 frame, + // Buffer1 keeps latest S1T0 frame. + // Buffer2 keeps latest S0T1 frame. + switch (next_pattern_) { + case kKey: + result[0] = KeyFrameConfig(); + result[1].Id(1).S(1).T(0).Reference(0).Update(1); + next_pattern_ = kDeltaT1; + break; + case kDeltaT1: + result[0].Id(2).S(0).T(1).Reference(0).Update(2); + result[1].Id(3).S(1).T(1).Reference(2).Reference(1); + next_pattern_ = kDeltaT0; + break; + case kDeltaT0: + result[0].Id(4).S(0).T(0).ReferenceAndUpdate(0); + result[1].Id(5).S(1).T(0).Reference(0).ReferenceAndUpdate(1); + next_pattern_ = kDeltaT1; + break; + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL2T2::OnEncodeDone( + LayerFrameConfig config) { + if (config.IsKeyframe()) { + config = KeyFrameConfig(); + } + + absl::optional<GenericFrameInfo> frame_info; + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + if (config.TemporalId() == 0) { + frame_info->part_of_chain = {config.SpatialId() == 0, true}; + } else { + frame_info->part_of_chain = {false, false}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2.h new file mode 100644 index 00000000000..dbf5036c1f6 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1T1 0 0 +// /| /| / +// S1T0 0-+-0-+-0 +// | | | | | ... +// S0T1 | 0 | 0 | +// |/ |/ |/ +// S0T0 0---0---0-- +// Time-> 0 1 2 3 4 +class ScalabilityStructureL2T2 : public ScalableVideoController { + public: + ~ScalabilityStructureL2T2() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKey, + kDeltaT1, + kDeltaT0, + }; + LayerFrameConfig KeyFrameConfig() const; + + FramePattern next_pattern_ = kKey; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key.cc new file mode 100644 index 00000000000..7409070d168 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +// decode targets: S0T0, S0T1, S1T0, S1T1 +constexpr DecodeTargetIndication kDtis[6][4] = { + {kSwitch, kSwitch, kSwitch, kSwitch}, // kKey, S0 + {kNotPresent, kNotPresent, kSwitch, kSwitch}, // kKey, S1 + {kNotPresent, kDiscardable, kNotPresent, kNotPresent}, // kDeltaT1, S0 + {kNotPresent, kNotPresent, kNotPresent, kDiscardable}, // kDeltaT1, S1 + {kSwitch, kSwitch, kNotPresent, kNotPresent}, // kDeltaT0, S0 + {kNotPresent, kNotPresent, kSwitch, kSwitch}, // kDeltaT0, S1 +}; + +} // namespace + +ScalabilityStructureL2T2Key::~ScalabilityStructureL2T2Key() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T2Key::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 2; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T2Key::DependencyStructure() + const { + FrameDependencyStructure structure; + structure.num_decode_targets = 4; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 0, 1, 1}; + structure.templates.resize(6); + auto& templates = structure.templates; + templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0}); + templates[1].S(0).T(0).Dtis("SS--").ChainDiffs({4, 3}).FrameDiffs({4}); + templates[2].S(0).T(1).Dtis("-D--").ChainDiffs({2, 1}).FrameDiffs({2}); + templates[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1}); + templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 4}).FrameDiffs({4}); + templates[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2}); + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T2Key::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).Keyframe().S(0).T(0).Update(0); +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL2T2Key::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKey; + } + std::vector<LayerFrameConfig> result(2); + + // Buffer0 keeps latest S0T0 frame, + // Buffer1 keeps latest S1T0 frame. + switch (next_pattern_) { + case kKey: + result[0] = KeyFrameConfig(); + result[1].Id(1).S(1).T(0).Reference(0).Update(1); + next_pattern_ = kDeltaT1; + break; + case kDeltaT1: + result[0].Id(2).S(0).T(1).Reference(0); + result[1].Id(3).S(1).T(1).Reference(1); + next_pattern_ = kDeltaT0; + break; + case kDeltaT0: + result[0].Id(4).S(0).T(0).ReferenceAndUpdate(0); + result[1].Id(5).S(1).T(0).ReferenceAndUpdate(1); + next_pattern_ = kDeltaT1; + break; + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL2T2Key::OnEncodeDone( + LayerFrameConfig config) { + if (config.IsKeyframe()) { + config = KeyFrameConfig(); + } + + absl::optional<GenericFrameInfo> frame_info; + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + if (config.IsKeyframe()) { + frame_info->part_of_chain = {true, true}; + } else if (config.TemporalId() == 0) { + frame_info->part_of_chain = {config.SpatialId() == 0, + config.SpatialId() == 1}; + } else { + frame_info->part_of_chain = {false, false}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h new file mode 100644 index 00000000000..9adfcbcd585 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1T1 0 0 +// / / / +// S1T0 0---0---0 +// | ... +// S0T1 | 0 0 +// |/ / / +// S0T0 0---0---0 +// Time-> 0 1 2 3 4 +class ScalabilityStructureL2T2Key : public ScalableVideoController { + public: + ~ScalabilityStructureL2T2Key() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKey, + kDeltaT1, + kDeltaT0, + }; + LayerFrameConfig KeyFrameConfig() const; + + FramePattern next_pattern_ = kKey; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.cc new file mode 100644 index 00000000000..6b79332de84 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.cc @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[6][4] = { + {kSwitch, kSwitch, kSwitch, kSwitch}, // kKey, S0T0 + {kNotPresent, kNotPresent, kSwitch, kSwitch}, // kKey, S1T0 + {kSwitch, kSwitch, kNotPresent, kNotPresent}, // kDelta0, S0T0 + {kNotPresent, kNotPresent, kNotPresent, kDiscardable}, // kDelta0, S1T1 + {kNotPresent, kDiscardable, kNotPresent, kNotPresent}, // kDelta1, S0T1 + {kNotPresent, kNotPresent, kSwitch, kSwitch}, // kDelta1, S1T0 +}; + +} // namespace + +ScalabilityStructureL2T2KeyShift::~ScalabilityStructureL2T2KeyShift() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL2T2KeyShift::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 2; + return result; +} + +FrameDependencyStructure ScalabilityStructureL2T2KeyShift::DependencyStructure() + const { + FrameDependencyStructure structure; + structure.num_decode_targets = 4; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 0, 1, 1}; + structure.templates.resize(7); + auto& templates = structure.templates; + templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0}); + templates[1].S(0).T(0).Dtis("SS--").ChainDiffs({2, 1}).FrameDiffs({2}); + templates[2].S(0).T(0).Dtis("SS--").ChainDiffs({4, 1}).FrameDiffs({4}); + templates[3].S(0).T(1).Dtis("-D--").ChainDiffs({2, 3}).FrameDiffs({2}); + templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1}); + templates[5].S(1).T(0).Dtis("--SS").ChainDiffs({3, 4}).FrameDiffs({4}); + templates[6].S(1).T(1).Dtis("---D").ChainDiffs({1, 2}).FrameDiffs({2}); + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL2T2KeyShift::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).Keyframe().S(0).T(0).Update(0); +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL2T2KeyShift::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKey; + } + std::vector<LayerFrameConfig> result(2); + + // Buffer0 keeps latest S0T0 frame, + // Buffer1 keeps latest S1T0 frame. + switch (next_pattern_) { + case kKey: + result[0] = KeyFrameConfig(); + result[1].Id(1).S(1).T(0).Reference(0).Update(1); + next_pattern_ = kDelta0; + break; + case kDelta0: + result[0].Id(2).S(0).T(0).ReferenceAndUpdate(0); + result[1].Id(3).S(1).T(1).Reference(1); + next_pattern_ = kDelta1; + break; + case kDelta1: + result[0].Id(4).S(0).T(1).Reference(0); + result[1].Id(5).S(1).T(0).ReferenceAndUpdate(1); + next_pattern_ = kDelta0; + break; + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL2T2KeyShift::OnEncodeDone( + LayerFrameConfig config) { + if (config.IsKeyframe()) { + config = KeyFrameConfig(); + } + + absl::optional<GenericFrameInfo> frame_info; + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + if (config.IsKeyframe()) { + frame_info->part_of_chain = {true, true}; + } else if (config.TemporalId() == 0) { + frame_info->part_of_chain = {config.SpatialId() == 0, + config.SpatialId() == 1}; + } else { + frame_info->part_of_chain = {false, false}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h new file mode 100644 index 00000000000..1b18bd7c173 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_SHIFT_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_SHIFT_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1T1 0 0 +// / / / +// S1T0 0---0---0 +// | ... +// S0T1 | 0 0 +// | / / +// S0T0 0-0---0-- +// Time-> 0 1 2 3 4 +class ScalabilityStructureL2T2KeyShift : public ScalableVideoController { + public: + ~ScalabilityStructureL2T2KeyShift() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKey, + kDelta0, + kDelta1, + }; + LayerFrameConfig KeyFrameConfig() const; + + FramePattern next_pattern_ = kKey; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_SHIFT_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t1.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t1.cc new file mode 100644 index 00000000000..1dd729c54dd --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t1.cc @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +constexpr DecodeTargetIndication kDtis[5][3] = { + {kSwitch, kSwitch, kSwitch}, // Key, S0 + {kNotPresent, kSwitch, kSwitch}, // Key, S1 + {kNotPresent, kNotPresent, kSwitch}, // Key and Delta, S2 + {kSwitch, kRequired, kRequired}, // Delta, S0 + {kNotPresent, kSwitch, kRequired}, // Delta, S1 +}; + +} // namespace + +ScalabilityStructureL3T1::~ScalabilityStructureL3T1() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL3T1::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 3; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureL3T1::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 3; + structure.num_chains = 3; + structure.decode_target_protected_by_chain = {0, 1, 2}; + auto& templates = structure.templates; + templates.resize(6); + templates[0].S(0).Dtis("SRR").ChainDiffs({3, 2, 1}).FrameDiffs({3}); + templates[1].S(0).Dtis("SSS").ChainDiffs({0, 0, 0}); + templates[2].S(1).Dtis("-SR").ChainDiffs({1, 1, 1}).FrameDiffs({3, 1}); + templates[3].S(1).Dtis("-SS").ChainDiffs({1, 1, 1}).FrameDiffs({1}); + templates[4].S(2).Dtis("--S").ChainDiffs({2, 1, 1}).FrameDiffs({3, 1}); + templates[5].S(2).Dtis("--S").ChainDiffs({2, 1, 1}).FrameDiffs({1}); + return structure; +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL3T1::NextFrameConfig(bool restart) { + std::vector<LayerFrameConfig> config(3); + + // Buffer i keeps latest frame for spatial layer i + if (restart || keyframe_) { + config[0].Id(0).S(0).Keyframe().Update(0); + config[1].Id(1).S(1).Update(1).Reference(0); + config[2].Id(2).S(2).Update(2).Reference(1); + keyframe_ = false; + } else { + config[0].Id(3).S(0).ReferenceAndUpdate(0); + config[1].Id(4).S(1).ReferenceAndUpdate(1).Reference(0); + config[2].Id(2).S(2).ReferenceAndUpdate(2).Reference(1); + } + return config; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL3T1::OnEncodeDone( + LayerFrameConfig config) { + absl::optional<GenericFrameInfo> frame_info; + if (config.IsKeyframe() && config.Id() != 0) { + // Encoder generated a key frame without asking to. + if (config.SpatialId() > 0) { + RTC_LOG(LS_WARNING) << "Unexpected spatial id " << config.SpatialId() + << " for key frame."; + } + config = LayerFrameConfig().Id(0).S(0).Keyframe().Update(0); + } + + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + frame_info->part_of_chain = {config.SpatialId() == 0, config.SpatialId() <= 1, + true}; + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t1.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t1.h new file mode 100644 index 00000000000..404860d08fa --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t1.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_ + +#include <vector> + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S2 0-0-0- +// | | | +// S1 0-0-0-... +// | | | +// S0 0-0-0- +// Time-> 0 1 2 +class ScalabilityStructureL3T1 : public ScalableVideoController { + public: + ~ScalabilityStructureL3T1() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t3.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t3.cc new file mode 100644 index 00000000000..6ac75da4509 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t3.cc @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; +constexpr auto kRequired = DecodeTargetIndication::kRequired; + +constexpr DecodeTargetIndication kDtis[12][9] = { + // Key, S0 + {kSwitch, kSwitch, kSwitch, // S0 + kSwitch, kSwitch, kSwitch, // S1 + kSwitch, kSwitch, kSwitch}, // S2 + // Key, S1 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kSwitch, kSwitch, kSwitch, // S1 + kSwitch, kSwitch, kSwitch}, // S2 + // Key, S2 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kSwitch, kSwitch, kSwitch}, // S2 + // Delta, S0T2 + {kNotPresent, kNotPresent, kDiscardable, // S0 + kNotPresent, kNotPresent, kRequired, // S1 + kNotPresent, kNotPresent, kRequired}, // S2 + // Delta, S1T2 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kDiscardable, // S1 + kNotPresent, kNotPresent, kRequired}, // S2 + // Delta, S2T2 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kNotPresent, kNotPresent, kDiscardable}, // S2 + // Delta, S0T1 + {kNotPresent, kDiscardable, kSwitch, // S0 + kNotPresent, kRequired, kRequired, // S1 + kNotPresent, kRequired, kRequired}, // S2 + // Delta, S1T1 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kDiscardable, kSwitch, // S1 + kNotPresent, kRequired, kRequired}, // S2 + // Delta, S2T1 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kNotPresent, kDiscardable, kSwitch}, // S2 + // Delta, S0T0 + {kSwitch, kSwitch, kSwitch, // S0 + kRequired, kRequired, kRequired, // S1 + kRequired, kRequired, kRequired}, // S2 + // Delta, S1T0 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kSwitch, kSwitch, kSwitch, // S1 + kRequired, kRequired, kRequired}, // S2 + // Delta, S2T0 + {kNotPresent, kNotPresent, kNotPresent, // S0 + kNotPresent, kNotPresent, kNotPresent, // S1 + kSwitch, kSwitch, kSwitch}, // S2 +}; + +} // namespace + +ScalabilityStructureL3T3::~ScalabilityStructureL3T3() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureL3T3::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 3; + result.num_temporal_layers = 3; + return result; +} + +FrameDependencyStructure ScalabilityStructureL3T3::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 9; + structure.num_chains = 3; + structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1, 2, 2, 2}; + auto& t = structure.templates; + t.resize(15); + // Templates are shown in the order frames following them appear in the + // stream, but in `structure.templates` array templates are sorted by + // (`spatial_id`, `temporal_id`) since that is a dependency descriptor + // requirement. Indexes are written in hex for nicer alignment. + t[0x1].S(0).T(0).Dtis("SSSSSSSSS").ChainDiffs({0, 0, 0}); + t[0x6].S(1).T(0).Dtis("---SSSSSS").ChainDiffs({1, 1, 1}).FrameDiffs({1}); + t[0xB].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 1}).FrameDiffs({1}); + t[0x3].S(0).T(2).Dtis("--D--R--R").ChainDiffs({3, 2, 1}).FrameDiffs({3}); + t[0x8].S(1).T(2).Dtis("-----D--R").ChainDiffs({4, 3, 2}).FrameDiffs({3, 1}); + t[0xD].S(2).T(2).Dtis("--------D").ChainDiffs({5, 4, 3}).FrameDiffs({3, 1}); + t[0x2].S(0).T(1).Dtis("-DS-RR-RR").ChainDiffs({6, 5, 4}).FrameDiffs({6}); + t[0x7].S(1).T(1).Dtis("----DS-RR").ChainDiffs({7, 6, 5}).FrameDiffs({6, 1}); + t[0xC].S(2).T(1).Dtis("-------DS").ChainDiffs({8, 7, 6}).FrameDiffs({6, 1}); + t[0x4].S(0).T(2).Dtis("--D--R--R").ChainDiffs({9, 8, 7}).FrameDiffs({3}); + t[0x9].S(1).T(2).Dtis("-----D--R").ChainDiffs({10, 9, 8}).FrameDiffs({3, 1}); + t[0xE].S(2).T(2).Dtis("--------D").ChainDiffs({11, 10, 9}).FrameDiffs({3, 1}); + t[0x0].S(0).T(0).Dtis("SSSRRRRRR").ChainDiffs({12, 11, 10}).FrameDiffs({12}); + t[0x5].S(1).T(0).Dtis("---SSSRRR").ChainDiffs({1, 1, 1}).FrameDiffs({12, 1}); + t[0xA].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 1}).FrameDiffs({12, 1}); + return structure; +} + +ScalableVideoController::LayerFrameConfig +ScalabilityStructureL3T3::KeyFrameConfig() const { + return LayerFrameConfig().Id(0).S(0).T(0).Keyframe().Update(0); +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureL3T3::NextFrameConfig(bool restart) { + if (restart) { + next_pattern_ = kKeyFrame; + } + std::vector<LayerFrameConfig> config(3); + + // For this structure name each of 8 buffers after the layer of the frame that + // buffer keeps. + static constexpr int kS0T0 = 0; + static constexpr int kS1T0 = 1; + static constexpr int kS2T0 = 2; + static constexpr int kS0T1 = 3; + static constexpr int kS1T1 = 4; + static constexpr int kS2T1 = 5; + static constexpr int kS0T2 = 6; + static constexpr int kS1T2 = 7; + switch (next_pattern_) { + case kKeyFrame: + config[0].Id(0).S(0).T(0).Keyframe().Update(kS0T0); + config[1].Id(1).S(1).T(0).Update(kS1T0).Reference(kS0T0); + config[2].Id(2).S(2).T(0).Update(kS2T0).Reference(kS1T0); + next_pattern_ = kDeltaFrameT2A; + break; + case kDeltaFrameT2A: + config[0].Id(3).S(0).T(2).Reference(kS0T0).Update(kS0T2); + config[1].Id(4).S(1).T(2).Reference(kS1T0).Reference(kS0T2).Update(kS1T2); + config[2].Id(5).S(2).T(2).Reference(kS2T0).Reference(kS1T2); + next_pattern_ = kDeltaFrameT1; + break; + case kDeltaFrameT1: + config[0].Id(6).S(0).T(1).Reference(kS0T0).Update(kS0T1); + config[1].Id(7).S(1).T(1).Reference(kS1T0).Reference(kS0T1).Update(kS1T1); + config[2].Id(8).S(2).T(1).Reference(kS2T0).Reference(kS1T1).Update(kS2T1); + next_pattern_ = kDeltaFrameT2B; + break; + case kDeltaFrameT2B: + config[0].Id(3).S(0).T(2).Reference(kS0T1).Update(kS0T2); + config[1].Id(4).S(1).T(2).Reference(kS1T1).Reference(kS0T2).Update(kS1T2); + config[2].Id(5).S(2).T(2).Reference(kS2T1).Reference(kS1T2); + next_pattern_ = kDeltaFrameT0; + break; + case kDeltaFrameT0: + config[0].Id(9).S(0).T(0).ReferenceAndUpdate(kS0T0); + config[1].Id(10).S(1).T(0).ReferenceAndUpdate(kS1T0).Reference(kS0T0); + config[2].Id(11).S(2).T(0).ReferenceAndUpdate(kS2T0).Reference(kS1T0); + next_pattern_ = kDeltaFrameT2A; + break; + } + return config; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureL3T3::OnEncodeDone( + LayerFrameConfig config) { + if (config.IsKeyframe() && config.Id() != 0) { + // Encoder generated a key frame without asking to. + if (config.SpatialId() > 0) { + RTC_LOG(LS_WARNING) << "Unexpected spatial id " << config.SpatialId() + << " for key frame."; + } + config = LayerFrameConfig() + .Keyframe() + .Id(0) + .S(0) + .T(0) + .Update(0) + .Update(1) + .Update(2) + .Update(3) + .Update(4) + .Update(5) + .Update(6) + .Update(7); + } + + absl::optional<GenericFrameInfo> frame_info; + if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = config.Buffers(); + frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]), + std::end(kDtis[config.Id()])); + if (config.TemporalId() == 0) { + frame_info->part_of_chain = {config.SpatialId() == 0, + config.SpatialId() <= 1, true}; + } else { + frame_info->part_of_chain = {false, false, false}; + } + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t3.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t3.h new file mode 100644 index 00000000000..363f07e015a --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_l3t3.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_ + +#include <vector> + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// https://aomediacodec.github.io/av1-rtp-spec/#a63-l3t3-full-svc +class ScalabilityStructureL3T3 : public ScalableVideoController { + public: + ~ScalabilityStructureL3T3() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + enum FramePattern { + kKeyFrame, + kDeltaFrameT2A, + kDeltaFrameT1, + kDeltaFrameT2B, + kDeltaFrameT0, + }; + LayerFrameConfig KeyFrameConfig() const; + + FramePattern next_pattern_ = kKeyFrame; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_s2t1.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_s2t1.cc new file mode 100644 index 00000000000..267363f0be4 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_s2t1.cc @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" + +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; + +constexpr DecodeTargetIndication kDtis[2][2] = { + {kSwitch, kNotPresent}, // S0 + {kNotPresent, kSwitch}, // S1 +}; + +} // namespace + +ScalabilityStructureS2T1::~ScalabilityStructureS2T1() = default; + +ScalableVideoController::StreamLayersConfig +ScalabilityStructureS2T1::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 2; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 2; + structure.num_chains = 2; + structure.decode_target_protected_by_chain = {0, 1}; + structure.templates.resize(4); + structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2}); + structure.templates[1].S(0).Dtis("S-").ChainDiffs({0, 0}); + structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2}); + structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 0}); + return structure; +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalabilityStructureS2T1::NextFrameConfig(bool restart) { + std::vector<LayerFrameConfig> result(2); + // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame. + if (restart || keyframe_) { + result[0].S(0).Keyframe().Update(0); + result[1].S(1).Keyframe().Update(1); + keyframe_ = false; + } else { + result[0].S(0).ReferenceAndUpdate(0); + result[1].S(1).ReferenceAndUpdate(1); + } + return result; +} + +absl::optional<GenericFrameInfo> ScalabilityStructureS2T1::OnEncodeDone( + LayerFrameConfig config) { + absl::optional<GenericFrameInfo> frame_info; + if (config.SpatialId() < 0 || + config.SpatialId() >= int{ABSL_ARRAYSIZE(kDtis)}) { + RTC_LOG(LS_ERROR) << "Unexpected spatial id " << config.SpatialId(); + return frame_info; + } + frame_info.emplace(); + frame_info->spatial_id = config.SpatialId(); + frame_info->temporal_id = config.TemporalId(); + frame_info->encoder_buffers = std::move(config.Buffers()); + frame_info->decode_target_indications.assign( + std::begin(kDtis[config.SpatialId()]), + std::end(kDtis[config.SpatialId()])); + frame_info->part_of_chain = {config.SpatialId() == 0, + config.SpatialId() == 1}; + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_s2t1.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_s2t1.h new file mode 100644 index 00000000000..06a99775c49 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_s2t1.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +// S1 0--0--0- +// ... +// S0 0--0--0- +class ScalabilityStructureS2T1 : public ScalableVideoController { + public: + ~ScalabilityStructureS2T1() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + bool keyframe_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_unittest.cc new file mode 100644 index 00000000000..d2a0516567a --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalability_structure_unittest.cc @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <stddef.h> +#include <stdint.h> + +#include <functional> +#include <memory> +#include <ostream> +#include <string> + +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/video_frame_type.h" +#include "modules/video_coding/chain_diff_calculator.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h" +#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h" +#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" +#include "modules/video_coding/frame_dependencies_calculator.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::AllOf; +using ::testing::Contains; +using ::testing::Each; +using ::testing::Field; +using ::testing::Ge; +using ::testing::IsEmpty; +using ::testing::Le; +using ::testing::Lt; +using ::testing::Not; +using ::testing::SizeIs; +using ::testing::TestWithParam; +using ::testing::Values; + +struct SvcTestParam { + friend std::ostream& operator<<(std::ostream& os, const SvcTestParam& param) { + return os << param.name; + } + + std::string name; + std::function<std::unique_ptr<ScalableVideoController>()> svc_factory; + int num_temporal_units; +}; + +class ScalabilityStructureTest : public TestWithParam<SvcTestParam> { + public: + std::vector<GenericFrameInfo> GenerateAllFrames() { + std::vector<GenericFrameInfo> frames; + + FrameDependenciesCalculator frame_deps_calculator; + ChainDiffCalculator chain_diff_calculator; + std::unique_ptr<ScalableVideoController> structure_controller = + GetParam().svc_factory(); + FrameDependencyStructure structure = + structure_controller->DependencyStructure(); + for (int i = 0; i < GetParam().num_temporal_units; ++i) { + for (auto& layer_frame : + structure_controller->NextFrameConfig(/*reset=*/false)) { + int64_t frame_id = static_cast<int64_t>(frames.size()); + bool is_keyframe = layer_frame.IsKeyframe(); + absl::optional<GenericFrameInfo> frame_info = + structure_controller->OnEncodeDone(std::move(layer_frame)); + EXPECT_TRUE(frame_info.has_value()); + if (is_keyframe) { + chain_diff_calculator.Reset(frame_info->part_of_chain); + } + frame_info->chain_diffs = + chain_diff_calculator.From(frame_id, frame_info->part_of_chain); + for (int64_t base_frame_id : frame_deps_calculator.FromBuffersUsage( + is_keyframe ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta, + frame_id, frame_info->encoder_buffers)) { + EXPECT_LT(base_frame_id, frame_id); + EXPECT_GE(base_frame_id, 0); + frame_info->frame_diffs.push_back(frame_id - base_frame_id); + } + + frames.push_back(*std::move(frame_info)); + } + } + return frames; + } +}; + +TEST_P(ScalabilityStructureTest, + NumberOfDecodeTargetsAndChainsAreInRangeAndConsistent) { + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + EXPECT_GT(structure.num_decode_targets, 0); + EXPECT_LE(structure.num_decode_targets, + DependencyDescriptor::kMaxDecodeTargets); + EXPECT_GE(structure.num_chains, 0); + EXPECT_LE(structure.num_chains, structure.num_decode_targets); + if (structure.num_chains == 0) { + EXPECT_THAT(structure.decode_target_protected_by_chain, IsEmpty()); + } else { + EXPECT_THAT(structure.decode_target_protected_by_chain, + AllOf(SizeIs(structure.num_decode_targets), Each(Ge(0)), + Each(Le(structure.num_chains)))); + } + EXPECT_THAT(structure.templates, + SizeIs(Lt(size_t{DependencyDescriptor::kMaxTemplates}))); +} + +TEST_P(ScalabilityStructureTest, TemplatesAreSortedByLayerId) { + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + ASSERT_THAT(structure.templates, Not(IsEmpty())); + const auto& first_templates = structure.templates.front(); + EXPECT_EQ(first_templates.spatial_id, 0); + EXPECT_EQ(first_templates.temporal_id, 0); + for (size_t i = 1; i < structure.templates.size(); ++i) { + const auto& prev_template = structure.templates[i - 1]; + const auto& next_template = structure.templates[i]; + if (next_template.spatial_id == prev_template.spatial_id && + next_template.temporal_id == prev_template.temporal_id) { + // Same layer, next_layer_idc == 0 + } else if (next_template.spatial_id == prev_template.spatial_id && + next_template.temporal_id == prev_template.temporal_id + 1) { + // Next temporal layer, next_layer_idc == 1 + } else if (next_template.spatial_id == prev_template.spatial_id + 1 && + next_template.temporal_id == 0) { + // Next spatial layer, next_layer_idc == 2 + } else { + // everything else is invalid. + ADD_FAILURE() << "Invalid templates order. Template #" << i + << " with layer (" << next_template.spatial_id << "," + << next_template.temporal_id + << ") follows template with layer (" + << prev_template.spatial_id << "," + << prev_template.temporal_id << ")."; + } + } +} + +TEST_P(ScalabilityStructureTest, TemplatesMatchNumberOfDecodeTargetsAndChains) { + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + EXPECT_THAT( + structure.templates, + Each(AllOf(Field(&FrameDependencyTemplate::decode_target_indications, + SizeIs(structure.num_decode_targets)), + Field(&FrameDependencyTemplate::chain_diffs, + SizeIs(structure.num_chains))))); +} + +TEST_P(ScalabilityStructureTest, FrameInfoMatchesFrameDependencyStructure) { + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames(); + for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) { + const auto& frame = frame_infos[frame_id]; + EXPECT_GE(frame.spatial_id, 0) << " for frame " << frame_id; + EXPECT_GE(frame.temporal_id, 0) << " for frame " << frame_id; + EXPECT_THAT(frame.decode_target_indications, + SizeIs(structure.num_decode_targets)) + << " for frame " << frame_id; + EXPECT_THAT(frame.part_of_chain, SizeIs(structure.num_chains)) + << " for frame " << frame_id; + } +} + +TEST_P(ScalabilityStructureTest, ThereIsAPerfectTemplateForEachFrame) { + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames(); + for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) { + EXPECT_THAT(structure.templates, Contains(frame_infos[frame_id])) + << " for frame " << frame_id; + } +} + +TEST_P(ScalabilityStructureTest, FrameDependsOnSameOrLowerLayer) { + std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames(); + int64_t num_frames = frame_infos.size(); + + for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) { + const auto& frame = frame_infos[frame_id]; + for (int frame_diff : frame.frame_diffs) { + int64_t base_frame_id = frame_id - frame_diff; + const auto& base_frame = frame_infos[base_frame_id]; + EXPECT_GE(frame.spatial_id, base_frame.spatial_id) + << "Frame " << frame_id << " depends on frame " << base_frame_id; + EXPECT_GE(frame.temporal_id, base_frame.temporal_id) + << "Frame " << frame_id << " depends on frame " << base_frame_id; + } + } +} + +TEST_P(ScalabilityStructureTest, NoFrameDependsOnDiscardableOrNotPresent) { + std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames(); + int64_t num_frames = frame_infos.size(); + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + + for (int dt = 0; dt < structure.num_decode_targets; ++dt) { + for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) { + const auto& frame = frame_infos[frame_id]; + if (frame.decode_target_indications[dt] == + DecodeTargetIndication::kNotPresent) { + continue; + } + for (int frame_diff : frame.frame_diffs) { + int64_t base_frame_id = frame_id - frame_diff; + const auto& base_frame = frame_infos[base_frame_id]; + EXPECT_NE(base_frame.decode_target_indications[dt], + DecodeTargetIndication::kNotPresent) + << "Frame " << frame_id << " depends on frame " << base_frame_id + << " that is not part of decode target#" << dt; + EXPECT_NE(base_frame.decode_target_indications[dt], + DecodeTargetIndication::kDiscardable) + << "Frame " << frame_id << " depends on frame " << base_frame_id + << " that is discardable for decode target#" << dt; + } + } + } +} + +TEST_P(ScalabilityStructureTest, NoFrameDependsThroughSwitchIndication) { + FrameDependencyStructure structure = + GetParam().svc_factory()->DependencyStructure(); + std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames(); + int64_t num_frames = frame_infos.size(); + std::vector<std::set<int64_t>> full_deps(num_frames); + + // For each frame calculate set of all frames it depends on, both directly and + // indirectly. + for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) { + std::set<int64_t> all_base_frames; + for (int frame_diff : frame_infos[frame_id].frame_diffs) { + int64_t base_frame_id = frame_id - frame_diff; + all_base_frames.insert(base_frame_id); + const auto& indirect = full_deps[base_frame_id]; + all_base_frames.insert(indirect.begin(), indirect.end()); + } + full_deps[frame_id] = std::move(all_base_frames); + } + + // Now check the switch indication: frames after the switch indication mustn't + // depend on any addition frames before the switch indications. + for (int dt = 0; dt < structure.num_decode_targets; ++dt) { + for (int64_t switch_frame_id = 0; switch_frame_id < num_frames; + ++switch_frame_id) { + if (frame_infos[switch_frame_id].decode_target_indications[dt] != + DecodeTargetIndication::kSwitch) { + continue; + } + for (int64_t later_frame_id = switch_frame_id + 1; + later_frame_id < num_frames; ++later_frame_id) { + if (frame_infos[later_frame_id].decode_target_indications[dt] == + DecodeTargetIndication::kNotPresent) { + continue; + } + for (int frame_diff : frame_infos[later_frame_id].frame_diffs) { + int64_t early_frame_id = later_frame_id - frame_diff; + if (early_frame_id < switch_frame_id) { + EXPECT_THAT(full_deps[switch_frame_id], Contains(early_frame_id)) + << "For decode target #" << dt << " frame " << later_frame_id + << " depends on the frame " << early_frame_id + << " that switch indication frame " << switch_frame_id + << " doesn't directly on indirectly depend on."; + } + } + } + } + } +} + +INSTANTIATE_TEST_SUITE_P( + Svc, + ScalabilityStructureTest, + Values(SvcTestParam{"L1T2", std::make_unique<ScalabilityStructureL1T2>, + /*num_temporal_units=*/4}, + SvcTestParam{"L1T3", std::make_unique<ScalabilityStructureL1T3>, + /*num_temporal_units=*/8}, + SvcTestParam{"L2T1", std::make_unique<ScalabilityStructureL2T1>, + /*num_temporal_units=*/3}, + SvcTestParam{"L2T1Key", + std::make_unique<ScalabilityStructureL2T1Key>, + /*num_temporal_units=*/3}, + SvcTestParam{"L3T1", std::make_unique<ScalabilityStructureL3T1>, + /*num_temporal_units=*/3}, + SvcTestParam{"L3T3", std::make_unique<ScalabilityStructureL3T3>, + /*num_temporal_units=*/8}, + SvcTestParam{"S2T1", std::make_unique<ScalabilityStructureS2T1>, + /*num_temporal_units=*/3}, + SvcTestParam{"L2T2", std::make_unique<ScalabilityStructureL2T2>, + /*num_temporal_units=*/4}, + SvcTestParam{"L2T2Key", + std::make_unique<ScalabilityStructureL2T2Key>, + /*num_temporal_units=*/4}, + SvcTestParam{"L2T2KeyShift", + std::make_unique<ScalabilityStructureL2T2KeyShift>, + /*num_temporal_units=*/4}), + [](const testing::TestParamInfo<SvcTestParam>& info) { + return info.param.name; + }); + +} // namespace +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller.h new file mode 100644 index 00000000000..d10aca2ce59 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_H_ + +#include <vector> + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/video_bitrate_allocation.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" + +namespace webrtc { + +// Controls how video should be encoded to be scalable. Outputs results as +// buffer usage configuration for encoder and enough details to communicate the +// scalability structure via dependency descriptor rtp header extension. +class ScalableVideoController { + public: + struct StreamLayersConfig { + int num_spatial_layers = 1; + int num_temporal_layers = 1; + }; + class LayerFrameConfig { + public: + // Builders/setters. + LayerFrameConfig& Id(int value); + LayerFrameConfig& Keyframe(); + LayerFrameConfig& S(int value); + LayerFrameConfig& T(int value); + LayerFrameConfig& Reference(int buffer_id); + LayerFrameConfig& Update(int buffer_id); + LayerFrameConfig& ReferenceAndUpdate(int buffer_id); + + // Getters. + int Id() const { return id_; } + bool IsKeyframe() const { return is_keyframe_; } + int SpatialId() const { return spatial_id_; } + int TemporalId() const { return temporal_id_; } + const absl::InlinedVector<CodecBufferUsage, kMaxEncoderBuffers>& Buffers() + const { + return buffers_; + } + + private: + // Id to match configuration returned by NextFrameConfig with + // (possibly modified) configuration passed back via OnEncoderDone. + // The meaning of the id is an implementation detail of + // the ScalableVideoController. + int id_ = 0; + + // Indication frame should be encoded as a key frame. In particular when + // `is_keyframe=true` property `CodecBufferUsage::referenced` should be + // ignored and treated as false. + bool is_keyframe_ = false; + + int spatial_id_ = 0; + int temporal_id_ = 0; + // Describes how encoder which buffers encoder allowed to reference and + // which buffers encoder should update. + absl::InlinedVector<CodecBufferUsage, kMaxEncoderBuffers> buffers_; + }; + + virtual ~ScalableVideoController() = default; + + // Returns video structure description for encoder to configure itself. + virtual StreamLayersConfig StreamConfig() const = 0; + + // Returns video structure description in format compatible with + // dependency descriptor rtp header extension. + virtual FrameDependencyStructure DependencyStructure() const = 0; + + // Notifies Controller with updated bitrates per layer. In particular notifies + // when certain layers should be disabled. + // Controller shouldn't produce LayerFrameConfig for disabled layers. + // TODO(bugs.webrtc.org/11404): Make pure virtual when implemented by all + // structures. + virtual void OnRatesUpdated(const VideoBitrateAllocation& bitrates) {} + + // When `restart` is true, first `LayerFrameConfig` should have `is_keyframe` + // set to true. + // Returned vector shouldn't be empty. + virtual std::vector<LayerFrameConfig> NextFrameConfig(bool restart) = 0; + + // Returns configuration to pass to EncoderCallback. + virtual absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) = 0; +}; + +// Below are implementation details. +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::Id(int value) { + id_ = value; + return *this; +} +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::Keyframe() { + is_keyframe_ = true; + return *this; +} +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::S(int value) { + spatial_id_ = value; + return *this; +} +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::T(int value) { + temporal_id_ = value; + return *this; +} +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::Reference(int buffer_id) { + buffers_.emplace_back(buffer_id, /*referenced=*/true, /*updated=*/false); + return *this; +} +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::Update(int buffer_id) { + buffers_.emplace_back(buffer_id, /*referenced=*/false, /*updated=*/true); + return *this; +} +inline ScalableVideoController::LayerFrameConfig& +ScalableVideoController::LayerFrameConfig::ReferenceAndUpdate(int buffer_id) { + buffers_.emplace_back(buffer_id, /*referenced=*/true, /*updated=*/true); + return *this; +} + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller_no_layering.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller_no_layering.cc new file mode 100644 index 00000000000..0d211fb9119 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller_no_layering.cc @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h" + +#include <utility> +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +ScalableVideoControllerNoLayering::~ScalableVideoControllerNoLayering() = + default; + +ScalableVideoController::StreamLayersConfig +ScalableVideoControllerNoLayering::StreamConfig() const { + StreamLayersConfig result; + result.num_spatial_layers = 1; + result.num_temporal_layers = 1; + return result; +} + +FrameDependencyStructure +ScalableVideoControllerNoLayering::DependencyStructure() const { + FrameDependencyStructure structure; + structure.num_decode_targets = 1; + FrameDependencyTemplate a_template; + a_template.decode_target_indications = {DecodeTargetIndication::kSwitch}; + structure.templates.push_back(a_template); + return structure; +} + +std::vector<ScalableVideoController::LayerFrameConfig> +ScalableVideoControllerNoLayering::NextFrameConfig(bool restart) { + std::vector<LayerFrameConfig> result(1); + if (restart || start_) { + result[0].Id(0).Keyframe().Update(0); + } else { + result[0].Id(0).ReferenceAndUpdate(0); + } + start_ = false; + return result; +} + +absl::optional<GenericFrameInfo> +ScalableVideoControllerNoLayering::OnEncodeDone(LayerFrameConfig config) { + RTC_DCHECK_EQ(config.Id(), 0); + absl::optional<GenericFrameInfo> frame_info(absl::in_place); + frame_info->encoder_buffers = config.Buffers(); + if (config.IsKeyframe()) { + for (auto& buffer : frame_info->encoder_buffers) { + buffer.referenced = false; + } + } + frame_info->decode_target_indications = {DecodeTargetIndication::kSwitch}; + return frame_info; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h new file mode 100644 index 00000000000..ad730989afb --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_NO_LAYERING_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_NO_LAYERING_H_ + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "modules/video_coding/codecs/av1/scalable_video_controller.h" + +namespace webrtc { + +class ScalableVideoControllerNoLayering : public ScalableVideoController { + public: + ~ScalableVideoControllerNoLayering() override; + + StreamLayersConfig StreamConfig() const override; + FrameDependencyStructure DependencyStructure() const override; + + std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override; + absl::optional<GenericFrameInfo> OnEncodeDone( + LayerFrameConfig config) override; + + private: + bool start_ = true; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_NO_LAYERING_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/test/encoded_video_frame_producer.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/test/encoded_video_frame_producer.cc new file mode 100644 index 00000000000..7dc387b8572 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/test/encoded_video_frame_producer.cc @@ -0,0 +1,78 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" + +#include <memory> +#include <vector> + +#include "api/test/create_frame_generator.h" +#include "api/test/frame_generator_interface.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/video_frame.h" +#include "api/video/video_frame_type.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +class EncoderCallback : public EncodedImageCallback { + public: + explicit EncoderCallback( + std::vector<EncodedVideoFrameProducer::EncodedFrame>& output_frames) + : output_frames_(output_frames) {} + + private: + Result OnEncodedImage( + const EncodedImage& encoded_image, + const CodecSpecificInfo* codec_specific_info, + const RTPFragmentationHeader* /*fragmentation*/) override { + output_frames_.push_back({encoded_image, *codec_specific_info}); + return Result(Result::Error::OK); + } + + std::vector<EncodedVideoFrameProducer::EncodedFrame>& output_frames_; +}; + +} // namespace + +std::vector<EncodedVideoFrameProducer::EncodedFrame> +EncodedVideoFrameProducer::Encode() { + std::unique_ptr<test::FrameGeneratorInterface> frame_buffer_generator = + test::CreateSquareFrameGenerator( + resolution_.Width(), resolution_.Height(), + test::FrameGeneratorInterface::OutputType::kI420, absl::nullopt); + + std::vector<EncodedFrame> encoded_frames; + EncoderCallback encoder_callback(encoded_frames); + RTC_CHECK_EQ(encoder_.RegisterEncodeCompleteCallback(&encoder_callback), + WEBRTC_VIDEO_CODEC_OK); + + uint32_t rtp_tick = 90000 / framerate_fps_; + std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameDelta}; + for (int i = 0; i < num_input_frames_; ++i) { + VideoFrame frame = + VideoFrame::Builder() + .set_video_frame_buffer(frame_buffer_generator->NextFrame().buffer) + .set_timestamp_rtp(rtp_timestamp_) + .build(); + rtp_timestamp_ += rtp_tick; + RTC_CHECK_EQ(encoder_.Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_OK); + } + + RTC_CHECK_EQ(encoder_.RegisterEncodeCompleteCallback(nullptr), + WEBRTC_VIDEO_CODEC_OK); + return encoded_frames; +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/test/encoded_video_frame_producer.h b/chromium/third_party/webrtc/modules/video_coding/codecs/test/encoded_video_frame_producer.h new file mode 100644 index 00000000000..757da02422a --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/test/encoded_video_frame_producer.h @@ -0,0 +1,74 @@ +/* + * Copyright 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_TEST_ENCODED_VIDEO_FRAME_PRODUCER_H_ +#define MODULES_VIDEO_CODING_CODECS_TEST_ENCODED_VIDEO_FRAME_PRODUCER_H_ + +#include <stdint.h> + +#include <vector> + +#include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/encoded_image.h" +#include "api/video_codecs/video_encoder.h" +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { + +// Wrapper around VideoEncoder::Encode for convenient input (generates frames) +// and output (returns encoded frames instead of passing them to callback) +class EncodedVideoFrameProducer { + public: + struct EncodedFrame { + EncodedImage encoded_image; + CodecSpecificInfo codec_specific_info; + }; + + // `encoder` should be initialized, but shouldn't have `EncoderCallback` set. + explicit EncodedVideoFrameProducer(VideoEncoder& encoder) + : encoder_(encoder) {} + EncodedVideoFrameProducer(const EncodedVideoFrameProducer&) = delete; + EncodedVideoFrameProducer& operator=(const EncodedVideoFrameProducer&) = + delete; + + // Number of the input frames to pass to the encoder. + EncodedVideoFrameProducer& SetNumInputFrames(int value); + // Resolution of the input frames. + EncodedVideoFrameProducer& SetResolution(RenderResolution value); + + // Generates input video frames and encodes them with `encoder` provided in + // the constructor. Returns frame passed to the `OnEncodedImage` by wraping + // `EncodedImageCallback` underneath. + std::vector<EncodedFrame> Encode(); + + private: + VideoEncoder& encoder_; + + uint32_t rtp_timestamp_ = 1000; + int num_input_frames_ = 1; + int framerate_fps_ = 30; + RenderResolution resolution_ = {320, 180}; +}; + +inline EncodedVideoFrameProducer& EncodedVideoFrameProducer::SetNumInputFrames( + int value) { + RTC_DCHECK_GT(value, 0); + num_input_frames_ = value; + return *this; +} + +inline EncodedVideoFrameProducer& EncodedVideoFrameProducer::SetResolution( + RenderResolution value) { + resolution_ = value; + return *this; +} + +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_CODECS_TEST_ENCODED_VIDEO_FRAME_PRODUCER_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index 7e92b360bd0..990db54321e 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -205,6 +205,9 @@ void VideoCodecTestFixtureImpl::Config::SetCodecSettings( codec_settings.VP9()->numberOfSpatialLayers = static_cast<uint8_t>(num_spatial_layers); break; + case kVideoCodecAV1: + codec_settings.qpMax = 63; + break; case kVideoCodecH264: codec_settings.H264()->frameDroppingOn = frame_dropper_on; codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval; diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_libaom.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_libaom.cc new file mode 100644 index 00000000000..45730aa09e4 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/test/videocodec_test_libaom.cc @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include <memory> +#include <vector> + +#include "api/test/create_videocodec_test_fixture.h" +#include "api/test/video/function_video_encoder_factory.h" +#include "api/video_codecs/sdp_video_format.h" +#include "media/base/media_constants.h" +#include "media/engine/internal_decoder_factory.h" +#include "media/engine/internal_encoder_factory.h" +#include "media/engine/simulcast_encoder_adapter.h" +#include "test/gtest.h" +#include "test/testsupport/file_utils.h" + +namespace webrtc { +namespace test { +namespace { +// Test clips settings. +constexpr int kCifWidth = 352; +constexpr int kCifHeight = 288; +constexpr int kNumFramesLong = 300; + +VideoCodecTestFixture::Config CreateConfig(std::string filename) { + VideoCodecTestFixture::Config config; + config.filename = filename; + config.filepath = ResourcePath(config.filename, "yuv"); + config.num_frames = kNumFramesLong; + config.use_single_core = true; + return config; +} + +TEST(VideoCodecTestLibaom, HighBitrateAV1) { + auto config = CreateConfig("foreman_cif"); + config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true, + kCifWidth, kCifHeight); + config.num_frames = kNumFramesLong; + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector<RateProfile> rate_profiles = {{500, 30, 0}}; + + std::vector<RateControlThresholds> rc_thresholds = { + {12, 1, 0, 1, 0.3, 0.1, 0, 1}}; + + std::vector<QualityThresholds> quality_thresholds = {{37, 34, 0.94, 0.92}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); +} + +TEST(VideoCodecTestLibaom, VeryLowBitrateAV1) { + auto config = CreateConfig("foreman_cif"); + config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true, + kCifWidth, kCifHeight); + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector<RateProfile> rate_profiles = {{50, 30, 0}}; + + std::vector<RateControlThresholds> rc_thresholds = { + {15, 8, 75, 2, 2, 2, 2, 1}}; + + std::vector<QualityThresholds> quality_thresholds = {{28, 25, 0.70, 0.62}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); +} + +#if !defined(WEBRTC_ANDROID) +constexpr int kHdWidth = 1280; +constexpr int kHdHeight = 720; +TEST(VideoCodecTestLibaom, HdAV1) { + auto config = CreateConfig("ConferenceMotion_1280_720_50"); + config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true, + kHdWidth, kHdHeight); + config.num_frames = kNumFramesLong; + auto fixture = CreateVideoCodecTestFixture(config); + + std::vector<RateProfile> rate_profiles = {{1000, 50, 0}}; + + std::vector<RateControlThresholds> rc_thresholds = { + {13, 3, 0, 1, 0.3, 0.1, 0, 1}}; + + std::vector<QualityThresholds> quality_thresholds = {{36, 32, 0.93, 0.87}}; + + fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr); +} +#endif + +} // namespace +} // namespace test +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 83ea450d886..b5652593ae2 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -608,58 +608,52 @@ FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure( FrameDependencyStructure template_structure; template_structure.num_decode_targets = num_layers; - using Builder = GenericFrameInfo::Builder; switch (num_layers) { case 1: { - template_structure.templates = { - Builder().T(0).Dtis("S").Build(), - Builder().T(0).Dtis("S").Fdiffs({1}).Build(), - }; + template_structure.templates.resize(2); + template_structure.templates[0].T(0).Dtis("S"); + template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1}); return template_structure; } case 2: { - template_structure.templates = { - Builder().T(0).Dtis("SS").Build(), - Builder().T(0).Dtis("SS").Fdiffs({2}).Build(), - Builder().T(0).Dtis("SR").Fdiffs({2}).Build(), - Builder().T(1).Dtis("-S").Fdiffs({1}).Build(), - Builder().T(1).Dtis("-D").Fdiffs({1, 2}).Build(), - }; + template_structure.templates.resize(5); + template_structure.templates[0].T(0).Dtis("SS"); + template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2}); + template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2}); + template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1}); + template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1}); return template_structure; } case 3: { if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { - template_structure.templates = { - Builder().T(0).Dtis("SSS").Build(), - Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(), - Builder().T(1).Dtis("-DR").Fdiffs({2}).Build(), - Builder().T(2).Dtis("--S").Fdiffs({1}).Build(), - Builder().T(2).Dtis("--D").Fdiffs({1, 2}).Build(), - }; + template_structure.templates.resize(5); + template_structure.templates[0].T(0).Dtis("SSS"); + template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); + template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2}); + template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1}); + template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1}); } else { - template_structure.templates = { - Builder().T(0).Dtis("SSS").Build(), - Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(), - Builder().T(0).Dtis("SRR").Fdiffs({4}).Build(), - Builder().T(1).Dtis("-SS").Fdiffs({2}).Build(), - Builder().T(1).Dtis("-DS").Fdiffs({2, 4}).Build(), - Builder().T(2).Dtis("--D").Fdiffs({1}).Build(), - Builder().T(2).Dtis("--D").Fdiffs({1, 3}).Build(), - }; + template_structure.templates.resize(7); + template_structure.templates[0].T(0).Dtis("SSS"); + template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4}); + template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4}); + template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2}); + template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2}); + template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1}); + template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1}); } return template_structure; } case 4: { - template_structure.templates = { - Builder().T(0).Dtis("SSSS").Build(), - Builder().T(0).Dtis("SSSS").Fdiffs({8}).Build(), - Builder().T(1).Dtis("-SRR").Fdiffs({4}).Build(), - Builder().T(1).Dtis("-SRR").Fdiffs({4, 8}).Build(), - Builder().T(2).Dtis("--SR").Fdiffs({2}).Build(), - Builder().T(2).Dtis("--SR").Fdiffs({2, 4}).Build(), - Builder().T(3).Dtis("---D").Fdiffs({1}).Build(), - Builder().T(3).Dtis("---D").Fdiffs({1, 3}).Build(), - }; + template_structure.templates.resize(8); + template_structure.templates[0].T(0).Dtis("SSSS"); + template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8}); + template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4}); + template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8}); + template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2}); + template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4}); + template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1}); + template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3}); return template_structure; } default: diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index 29cfcf0489a..d127d8056d0 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -75,7 +75,7 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { DependencyInfo(absl::string_view indication_symbols, Vp8FrameConfig frame_config) : decode_target_indications( - GenericFrameInfo::DecodeTargetInfo(indication_symbols)), + webrtc_impl::StringToDecodeTargetIndications(indication_symbols)), frame_config(frame_config) {} absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications; diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc index a3ee2c0c41d..d86d8767c5d 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc @@ -44,26 +44,48 @@ constexpr int kVp8ErrorPropagationTh = 30; constexpr long kDecodeDeadlineRealtime = 1; // NOLINT const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm"; +const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config"; -void GetPostProcParamsFromFieldTrialGroup( - LibvpxVp8Decoder::DeblockParams* deblock_params) { - std::string group = - webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial); - if (group.empty()) - return; +#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ + defined(WEBRTC_ANDROID) +constexpr bool kIsArm = true; +#else +constexpr bool kIsArm = false; +#endif + +absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() { + if (kIsArm) { + // For ARM, this is only called when deblocking is explicitly enabled, and + // the default strength is set by the ctor. + return LibvpxVp8Decoder::DeblockParams(); + } + // For non-arm, don't use the explicit deblocking settings by default. + return absl::nullopt; +} + +absl::optional<LibvpxVp8Decoder::DeblockParams> +GetPostProcParamsFromFieldTrialGroup() { + std::string group = webrtc::field_trial::FindFullName( + kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial); + if (group.empty()) { + return DefaultDeblockParams(); + } LibvpxVp8Decoder::DeblockParams params; if (sscanf(group.c_str(), "Enabled-%d,%d,%d", ¶ms.max_level, - ¶ms.min_qp, ¶ms.degrade_qp) != 3) - return; + ¶ms.min_qp, ¶ms.degrade_qp) != 3) { + return DefaultDeblockParams(); + } - if (params.max_level < 0 || params.max_level > 16) - return; + if (params.max_level < 0 || params.max_level > 16) { + return DefaultDeblockParams(); + } - if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) - return; + if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) { + return DefaultDeblockParams(); + } - *deblock_params = params; + return params; } } // namespace @@ -97,8 +119,9 @@ class LibvpxVp8Decoder::QpSmoother { }; LibvpxVp8Decoder::LibvpxVp8Decoder() - : use_postproc_arm_( - webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)), + : use_postproc_( + kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial) + : true), buffer_pool_(false, 300 /* max_number_of_buffers*/), decode_complete_callback_(NULL), inited_(false), @@ -107,10 +130,9 @@ LibvpxVp8Decoder::LibvpxVp8Decoder() last_frame_width_(0), last_frame_height_(0), key_frame_required_(true), - qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) { - if (use_postproc_arm_) - GetPostProcParamsFromFieldTrialGroup(&deblock_); -} + deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup() + : absl::nullopt), + qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {} LibvpxVp8Decoder::~LibvpxVp8Decoder() { inited_ = true; // in order to do the actual release @@ -131,12 +153,7 @@ int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) { cfg.threads = 1; cfg.h = cfg.w = 0; // set after decode -#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ - defined(WEBRTC_ANDROID) - vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0; -#else - vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC; -#endif + vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0; if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) { delete decoder_; @@ -174,43 +191,47 @@ int LibvpxVp8Decoder::Decode(const EncodedImage& input_image, } // Post process configurations. -#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ - defined(WEBRTC_ANDROID) - if (use_postproc_arm_) { + if (use_postproc_) { vp8_postproc_cfg_t ppcfg; + // MFQE enabled to reduce key frame popping. ppcfg.post_proc_flag = VP8_MFQE; - // For low resolutions, use stronger deblocking filter. - int last_width_x_height = last_frame_width_ * last_frame_height_; - if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) { - // Enable the deblock and demacroblocker based on qp thresholds. - RTC_DCHECK(qp_smoother_); - int qp = qp_smoother_->GetAvg(); - if (qp > deblock_.min_qp) { - int level = deblock_.max_level; - if (qp < deblock_.degrade_qp) { - // Use lower level. - level = deblock_.max_level * (qp - deblock_.min_qp) / - (deblock_.degrade_qp - deblock_.min_qp); + + if (kIsArm) { + RTC_DCHECK(deblock_params_.has_value()); + } + if (deblock_params_.has_value()) { + // For low resolutions, use stronger deblocking filter. + int last_width_x_height = last_frame_width_ * last_frame_height_; + if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) { + // Enable the deblock and demacroblocker based on qp thresholds. + RTC_DCHECK(qp_smoother_); + int qp = qp_smoother_->GetAvg(); + if (qp > deblock_params_->min_qp) { + int level = deblock_params_->max_level; + if (qp < deblock_params_->degrade_qp) { + // Use lower level. + level = deblock_params_->max_level * + (qp - deblock_params_->min_qp) / + (deblock_params_->degrade_qp - deblock_params_->min_qp); + } + // Deblocking level only affects VP8_DEMACROBLOCK. + ppcfg.deblocking_level = std::max(level, 1); + ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK; } - // Deblocking level only affects VP8_DEMACROBLOCK. - ppcfg.deblocking_level = std::max(level, 1); - ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK; } + } else { + // Non-arm with no explicit deblock params set. + ppcfg.post_proc_flag |= VP8_DEBLOCK; + // For VGA resolutions and lower, enable the demacroblocker postproc. + if (last_frame_width_ * last_frame_height_ <= 640 * 360) { + ppcfg.post_proc_flag |= VP8_DEMACROBLOCK; + } + // Strength of deblocking filter. Valid range:[0,16] + ppcfg.deblocking_level = 3; } + vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); } -#else - vp8_postproc_cfg_t ppcfg; - // MFQE enabled to reduce key frame popping. - ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK; - // For VGA resolutions and lower, enable the demacroblocker postproc. - if (last_frame_width_ * last_frame_height_ <= 640 * 360) { - ppcfg.post_proc_flag |= VP8_DEMACROBLOCK; - } - // Strength of deblocking filter. Valid range:[0,16] - ppcfg.deblocking_level = 3; - vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg); -#endif // Always start with a complete key frame. if (key_frame_required_) { diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h index d9bfee81c1a..2a0c5f2c5ba 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h @@ -13,6 +13,7 @@ #include <memory> +#include "absl/types/optional.h" #include "api/video/encoded_image.h" #include "api/video_codecs/video_decoder.h" #include "common_video/include/i420_buffer_pool.h" @@ -51,7 +52,7 @@ class LibvpxVp8Decoder : public VideoDecoder { uint32_t timeStamp, int qp, const webrtc::ColorSpace* explicit_color_space); - const bool use_postproc_arm_; + const bool use_postproc_; I420BufferPool buffer_pool_; DecodedImageCallback* decode_complete_callback_; @@ -61,7 +62,7 @@ class LibvpxVp8Decoder : public VideoDecoder { int last_frame_width_; int last_frame_height_; bool key_frame_required_; - DeblockParams deblock_; + const absl::optional<DeblockParams> deblock_params_; const std::unique_ptr<QpSmoother> qp_smoother_; }; diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 01858c6ee9d..caccb4246c1 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -36,6 +36,7 @@ constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2; constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4; constexpr int kQpDeltaThresholdForSync = 8; constexpr int kMinBitrateKbpsForQpBoost = 500; +constexpr auto kSwitch = DecodeTargetIndication::kSwitch; } // namespace const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5; @@ -319,8 +320,7 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index, if (number_of_temporal_layers_ == 1) { vp8_info.temporalIdx = kNoTemporalIdx; vp8_info.layerSync = false; - generic_frame_info.decode_target_indications = - GenericFrameInfo::DecodeTargetInfo("S"); + generic_frame_info.decode_target_indications = {kSwitch}; generic_frame_info.encoder_buffers.emplace_back( 0, /*referenced=*/!is_keyframe, /*updated=*/true); } else { @@ -344,8 +344,7 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index, active_layer_ = 1; info->template_structure = GetTemplateStructure(number_of_temporal_layers_); - generic_frame_info.decode_target_indications = - GenericFrameInfo::DecodeTargetInfo("SS"); + generic_frame_info.decode_target_indications = {kSwitch, kSwitch}; } else if (active_layer_ >= 0 && layers_[active_layer_].state == TemporalLayer::State::kKeyFrame) { layers_[active_layer_].state = TemporalLayer::State::kNormal; @@ -429,21 +428,18 @@ FrameDependencyStructure ScreenshareLayers::GetTemplateStructure( FrameDependencyStructure template_structure; template_structure.num_decode_targets = num_layers; - using Builder = GenericFrameInfo::Builder; switch (num_layers) { case 1: { - template_structure.templates = { - Builder().T(0).Dtis("S").Build(), - Builder().T(0).Dtis("S").Fdiffs({1}).Build(), - }; + template_structure.templates.resize(2); + template_structure.templates[0].T(0).Dtis("S"); + template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1}); return template_structure; } case 2: { - template_structure.templates = { - Builder().T(0).Dtis("SS").Build(), - Builder().T(0).Dtis("SS").Fdiffs({1}).Build(), - Builder().T(1).Dtis("-S").Fdiffs({1}).Build(), - }; + template_structure.templates.resize(3); + template_structure.templates[0].T(0).Dtis("SS"); + template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({1}); + template_structure.templates[2].T(1).Dtis("-S").FrameDiffs({1}); return template_structure; } default: diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index 5270ffe81cb..39477f12f17 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -78,7 +78,7 @@ class ScreenshareLayers final : public Vp8FrameBufferController { DependencyInfo(absl::string_view indication_symbols, Vp8FrameConfig frame_config) : decode_target_indications( - GenericFrameInfo::DecodeTargetInfo(indication_symbols)), + webrtc_impl::StringToDecodeTargetIndications(indication_symbols)), frame_config(frame_config) {} absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications; diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h index dcff1e6a188..1ac927d29f9 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h @@ -19,74 +19,86 @@ namespace webrtc { class MockLibvpxVp8Interface : public LibvpxInterface { public: - MOCK_CONST_METHOD5(img_alloc, - vpx_image_t*(vpx_image_t*, - vpx_img_fmt_t, - unsigned int, - unsigned int, - unsigned int)); - MOCK_CONST_METHOD6(img_wrap, - vpx_image_t*(vpx_image_t*, - vpx_img_fmt_t, - unsigned int, - unsigned int, - unsigned int, - unsigned char*)); - MOCK_CONST_METHOD1(img_free, void(vpx_image_t* img)); - MOCK_CONST_METHOD2(codec_enc_config_set, - vpx_codec_err_t(vpx_codec_ctx_t*, - const vpx_codec_enc_cfg_t*)); - MOCK_CONST_METHOD3(codec_enc_config_default, - vpx_codec_err_t(vpx_codec_iface_t*, - vpx_codec_enc_cfg_t*, - unsigned int)); - MOCK_CONST_METHOD4(codec_enc_init, - vpx_codec_err_t(vpx_codec_ctx_t*, - vpx_codec_iface_t*, - const vpx_codec_enc_cfg_t*, - vpx_codec_flags_t)); - MOCK_CONST_METHOD6(codec_enc_init_multi, - vpx_codec_err_t(vpx_codec_ctx_t*, - vpx_codec_iface_t*, - vpx_codec_enc_cfg_t*, - int, - vpx_codec_flags_t, - vpx_rational_t*)); - MOCK_CONST_METHOD1(codec_destroy, vpx_codec_err_t(vpx_codec_ctx_t*)); - MOCK_CONST_METHOD3(codec_control, - vpx_codec_err_t(vpx_codec_ctx_t*, - vp8e_enc_control_id, - uint32_t)); - MOCK_CONST_METHOD3(codec_control, - vpx_codec_err_t(vpx_codec_ctx_t*, - vp8e_enc_control_id, - int)); - MOCK_CONST_METHOD3(codec_control, - vpx_codec_err_t(vpx_codec_ctx_t*, - vp8e_enc_control_id, - int*)); - MOCK_CONST_METHOD3(codec_control, - vpx_codec_err_t(vpx_codec_ctx_t*, - vp8e_enc_control_id, - vpx_roi_map*)); - MOCK_CONST_METHOD3(codec_control, - vpx_codec_err_t(vpx_codec_ctx_t*, - vp8e_enc_control_id, - vpx_active_map*)); - MOCK_CONST_METHOD3(codec_control, - vpx_codec_err_t(vpx_codec_ctx_t*, - vp8e_enc_control_id, - vpx_scaling_mode*)); - MOCK_CONST_METHOD6(codec_encode, - vpx_codec_err_t(vpx_codec_ctx_t*, - const vpx_image_t*, - vpx_codec_pts_t, - uint64_t, - vpx_enc_frame_flags_t, - uint64_t)); - MOCK_CONST_METHOD2(codec_get_cx_data, - const vpx_codec_cx_pkt_t*(vpx_codec_ctx_t*, - vpx_codec_iter_t*)); + MOCK_METHOD( + vpx_image_t*, + img_alloc, + (vpx_image_t*, vpx_img_fmt_t, unsigned int, unsigned int, unsigned int), + (const, override)); + MOCK_METHOD(vpx_image_t*, + img_wrap, + (vpx_image_t*, + vpx_img_fmt_t, + unsigned int, + unsigned int, + unsigned int, + unsigned char*), + (const, override)); + MOCK_METHOD(void, img_free, (vpx_image_t * img), (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_enc_config_set, + (vpx_codec_ctx_t*, const vpx_codec_enc_cfg_t*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_enc_config_default, + (vpx_codec_iface_t*, vpx_codec_enc_cfg_t*, unsigned int), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_enc_init, + (vpx_codec_ctx_t*, + vpx_codec_iface_t*, + const vpx_codec_enc_cfg_t*, + vpx_codec_flags_t), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_enc_init_multi, + (vpx_codec_ctx_t*, + vpx_codec_iface_t*, + vpx_codec_enc_cfg_t*, + int, + vpx_codec_flags_t, + vpx_rational_t*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_destroy, + (vpx_codec_ctx_t*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_control, + (vpx_codec_ctx_t*, vp8e_enc_control_id, uint32_t), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_control, + (vpx_codec_ctx_t*, vp8e_enc_control_id, int), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_control, + (vpx_codec_ctx_t*, vp8e_enc_control_id, int*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_control, + (vpx_codec_ctx_t*, vp8e_enc_control_id, vpx_roi_map*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_control, + (vpx_codec_ctx_t*, vp8e_enc_control_id, vpx_active_map*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_control, + (vpx_codec_ctx_t*, vp8e_enc_control_id, vpx_scaling_mode*), + (const, override)); + MOCK_METHOD(vpx_codec_err_t, + codec_encode, + (vpx_codec_ctx_t*, + const vpx_image_t*, + vpx_codec_pts_t, + uint64_t, + vpx_enc_frame_flags_t, + uint64_t), + (const, override)); + MOCK_METHOD(const vpx_codec_cx_pkt_t*, + codec_get_cx_data, + (vpx_codec_ctx_t*, vpx_codec_iter_t*), + (const, override)); }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/include/vp9.h b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/include/vp9.h index 8091cacec90..7cf1c2ebd1b 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/include/vp9.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/include/vp9.h @@ -25,6 +25,10 @@ namespace webrtc { // negotiate in SDP, in order of preference. std::vector<SdpVideoFormat> SupportedVP9Codecs(); +// Returns a vector with all supported internal VP9 decode profiles in order of +// preference. These will be availble for receive-only connections. +std::vector<SdpVideoFormat> SupportedVP9DecoderCodecs(); + class VP9Encoder : public VideoEncoder { public: // Deprecated. Returns default implementation using VP9 Profile 0. diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index d40cf23257f..4d5b8497d1a 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -16,6 +16,7 @@ #include "common_video/libyuv/include/webrtc_libyuv.h" #include "media/base/vp9_profile.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" +#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h" #include "modules/video_coding/codecs/test/video_codec_unittest.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" #include "modules/video_coding/codecs/vp9/svc_config.h" @@ -25,20 +26,33 @@ #include "test/video_codec_settings.h" namespace webrtc { +namespace { using ::testing::ElementsAreArray; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAreArray; using EncoderInfo = webrtc::VideoEncoder::EncoderInfo; using FramerateFractions = absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>; -namespace { -const size_t kWidth = 1280; -const size_t kHeight = 720; +constexpr size_t kWidth = 1280; +constexpr size_t kHeight = 720; const VideoEncoder::Capabilities kCapabilities(false); const VideoEncoder::Settings kSettings(kCapabilities, /*number_of_cores=*/1, /*max_payload_size=*/0); + +VideoCodec DefaultCodecSettings() { + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP9, &codec_settings); + codec_settings.width = kWidth; + codec_settings.height = kHeight; + codec_settings.VP9()->numberOfTemporalLayers = 1; + codec_settings.VP9()->numberOfSpatialLayers = 1; + return codec_settings; +} + } // namespace class TestVp9Impl : public VideoCodecUnitTest { @@ -59,53 +73,6 @@ class TestVp9Impl : public VideoCodecUnitTest { codec_settings->VP9()->numberOfSpatialLayers = 1; } - void ExpectFrameWith(uint8_t temporal_idx) { - EncodedImage encoded_frame; - CodecSpecificInfo codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); - EXPECT_EQ(temporal_idx, codec_specific_info.codecSpecific.VP9.temporal_idx); - } - - void ExpectFrameWith(size_t num_spatial_layers, - uint8_t temporal_idx, - bool temporal_up_switch, - uint8_t num_ref_pics, - const std::vector<uint8_t>& p_diff) { - std::vector<EncodedImage> encoded_frame; - std::vector<CodecSpecificInfo> codec_specific; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific)); - for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers; - ++spatial_idx) { - const CodecSpecificInfoVP9& vp9 = - codec_specific[spatial_idx].codecSpecific.VP9; - if (vp9.temporal_idx == kNoTemporalIdx) { - EXPECT_EQ(temporal_idx, 0); - } else { - EXPECT_EQ(vp9.temporal_idx, temporal_idx); - } - if (num_spatial_layers == 1) { - EXPECT_FALSE(encoded_frame[spatial_idx].SpatialIndex()); - } else { - EXPECT_EQ(encoded_frame[spatial_idx].SpatialIndex(), - static_cast<int>(spatial_idx)); - } - EXPECT_EQ(vp9.temporal_up_switch, temporal_up_switch); - - // Ensure there are no duplicates in reference list. - std::vector<uint8_t> vp9_p_diff(vp9.p_diff, - vp9.p_diff + vp9.num_ref_pics); - std::sort(vp9_p_diff.begin(), vp9_p_diff.end()); - EXPECT_EQ(std::unique(vp9_p_diff.begin(), vp9_p_diff.end()), - vp9_p_diff.end()); - - for (size_t ref_pic_num = 0; ref_pic_num < num_ref_pics; ++ref_pic_num) { - EXPECT_NE( - std::find(p_diff.begin(), p_diff.end(), vp9.p_diff[ref_pic_num]), - p_diff.end()); - } - } - } - void ConfigureSvc(size_t num_spatial_layers, size_t num_temporal_layers = 1) { codec_settings_.VP9()->numberOfSpatialLayers = static_cast<unsigned char>(num_spatial_layers); @@ -187,57 +154,61 @@ TEST_F(TestVp9Impl, DecodedQpEqualsEncodedQp) { EXPECT_EQ(encoded_frame.qp_, *decoded_qp); } -TEST_F(TestVp9Impl, ParserQpEqualsEncodedQp) { - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); - EncodedImage encoded_frame; - CodecSpecificInfo codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); - +TEST(Vp9ImplTest, ParserQpEqualsEncodedQp) { + std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create(); + VideoCodec codec_settings = DefaultCodecSettings(); + encoder->InitEncode(&codec_settings, kSettings); + + std::vector<EncodedVideoFrameProducer::EncodedFrame> frames = + EncodedVideoFrameProducer(*encoder) + .SetNumInputFrames(1) + .SetResolution({kWidth, kHeight}) + .Encode(); + ASSERT_THAT(frames, SizeIs(1)); + const auto& encoded_frame = frames.front().encoded_image; int qp = 0; ASSERT_TRUE(vp9::GetQp(encoded_frame.data(), encoded_frame.size(), &qp)); EXPECT_EQ(encoded_frame.qp_, qp); } -TEST_F(TestVp9Impl, EncoderWith2TemporalLayers) { - // Override default settings. - codec_settings_.VP9()->numberOfTemporalLayers = 2; +TEST(Vp9ImplTest, EncoderWith2TemporalLayers) { + std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create(); + VideoCodec codec_settings = DefaultCodecSettings(); + codec_settings.VP9()->numberOfTemporalLayers = 2; // Tl0PidIdx is only used in non-flexible mode. - codec_settings_.VP9()->flexibleMode = false; - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->InitEncode(&codec_settings_, kSettings)); - - // Temporal layer 0. - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); - EncodedImage encoded_frame; - CodecSpecificInfo codec_specific_info; - ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info)); - EXPECT_EQ(0, codec_specific_info.codecSpecific.VP9.temporal_idx); - - // Temporal layer 1. - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); - ExpectFrameWith(1); - - // Temporal layer 0. - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); - ExpectFrameWith(0); - - // Temporal layer 1. - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); - ExpectFrameWith(1); + codec_settings.VP9()->flexibleMode = false; + EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings), + WEBRTC_VIDEO_CODEC_OK); + + std::vector<EncodedVideoFrameProducer::EncodedFrame> frames = + EncodedVideoFrameProducer(*encoder) + .SetNumInputFrames(4) + .SetResolution({kWidth, kHeight}) + .Encode(); + + ASSERT_THAT(frames, SizeIs(4)); + EXPECT_EQ(frames[0].codec_specific_info.codecSpecific.VP9.temporal_idx, 0); + EXPECT_EQ(frames[1].codec_specific_info.codecSpecific.VP9.temporal_idx, 1); + EXPECT_EQ(frames[2].codec_specific_info.codecSpecific.VP9.temporal_idx, 0); + EXPECT_EQ(frames[3].codec_specific_info.codecSpecific.VP9.temporal_idx, 1); } -TEST_F(TestVp9Impl, EncoderWith2SpatialLayers) { - codec_settings_.VP9()->numberOfSpatialLayers = 2; - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->InitEncode(&codec_settings_, kSettings)); - - SetWaitForEncodedFramesThreshold(2); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr)); - std::vector<EncodedImage> encoded_frame; - std::vector<CodecSpecificInfo> codec_info; - ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_info)); - EXPECT_EQ(encoded_frame[0].SpatialIndex(), 0); - EXPECT_EQ(encoded_frame[1].SpatialIndex(), 1); +TEST(Vp9ImplTest, EncoderWith2SpatialLayers) { + std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create(); + VideoCodec codec_settings = DefaultCodecSettings(); + codec_settings.VP9()->numberOfSpatialLayers = 2; + EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings), + WEBRTC_VIDEO_CODEC_OK); + + std::vector<EncodedVideoFrameProducer::EncodedFrame> frames = + EncodedVideoFrameProducer(*encoder) + .SetNumInputFrames(1) + .SetResolution({kWidth, kHeight}) + .Encode(); + + ASSERT_THAT(frames, SizeIs(2)); + EXPECT_EQ(frames[0].encoded_image.SpatialIndex(), 0); + EXPECT_EQ(frames[1].encoded_image.SpatialIndex(), 1); } TEST_F(TestVp9Impl, EncoderExplicitLayering) { @@ -1421,29 +1392,34 @@ TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) { ::testing::ElementsAreArray(expected_fps_allocation)); } -class TestVp9ImplWithLayering - : public TestVp9Impl, - public ::testing::WithParamInterface<::testing::tuple<uint8_t, uint8_t>> { +class Vp9ImplWithLayeringTest + : public ::testing::TestWithParam<std::tuple<int, int, bool>> { protected: - TestVp9ImplWithLayering() - : num_spatial_layers_(::testing::get<0>(GetParam())), - num_temporal_layers_(::testing::get<1>(GetParam())) {} + Vp9ImplWithLayeringTest() + : num_spatial_layers_(std::get<0>(GetParam())), + num_temporal_layers_(std::get<1>(GetParam())), + override_field_trials_(std::get<2>(GetParam()) + ? "WebRTC-Vp9ExternalRefCtrl/Enabled/" + : "") {} const uint8_t num_spatial_layers_; const uint8_t num_temporal_layers_; + const test::ScopedFieldTrials override_field_trials_; }; -TEST_P(TestVp9ImplWithLayering, FlexibleMode) { +TEST_P(Vp9ImplWithLayeringTest, FlexibleMode) { // In flexible mode encoder wrapper obtains actual list of references from // encoder and writes it into RTP payload descriptor. Check that reference // list in payload descriptor matches the predefined one, which is used // in non-flexible mode. - codec_settings_.VP9()->flexibleMode = true; - codec_settings_.VP9()->frameDroppingOn = false; - codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers_; - codec_settings_.VP9()->numberOfTemporalLayers = num_temporal_layers_; - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->InitEncode(&codec_settings_, kSettings)); + std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create(); + VideoCodec codec_settings = DefaultCodecSettings(); + codec_settings.VP9()->flexibleMode = true; + codec_settings.VP9()->frameDroppingOn = false; + codec_settings.VP9()->numberOfSpatialLayers = num_spatial_layers_; + codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers_; + EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings), + WEBRTC_VIDEO_CODEC_OK); GofInfoVP9 gof; if (num_temporal_layers_ == 1) { @@ -1456,65 +1432,48 @@ TEST_P(TestVp9ImplWithLayering, FlexibleMode) { // Encode at least (num_frames_in_gof + 1) frames to verify references // of non-key frame with gof_idx = 0. - for (size_t frame_num = 0; frame_num < gof.num_frames_in_gof + 1; - ++frame_num) { - SetWaitForEncodedFramesThreshold(num_spatial_layers_); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(NextInputFrame(), nullptr)); - - const bool is_key_frame = frame_num == 0; - const size_t gof_idx = frame_num % gof.num_frames_in_gof; - const std::vector<uint8_t> p_diff(std::begin(gof.pid_diff[gof_idx]), - std::end(gof.pid_diff[gof_idx])); - - ExpectFrameWith(num_spatial_layers_, gof.temporal_idx[gof_idx], - gof.temporal_up_switch[gof_idx], - is_key_frame ? 0 : gof.num_ref_pics[gof_idx], p_diff); - } -} - -TEST_P(TestVp9ImplWithLayering, ExternalRefControl) { - test::ScopedFieldTrials override_field_trials( - "WebRTC-Vp9ExternalRefCtrl/Enabled/"); - codec_settings_.VP9()->flexibleMode = true; - codec_settings_.VP9()->frameDroppingOn = false; - codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers_; - codec_settings_.VP9()->numberOfTemporalLayers = num_temporal_layers_; - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->InitEncode(&codec_settings_, kSettings)); - - GofInfoVP9 gof; - if (num_temporal_layers_ == 1) { - gof.SetGofInfoVP9(kTemporalStructureMode1); - } else if (num_temporal_layers_ == 2) { - gof.SetGofInfoVP9(kTemporalStructureMode2); - } else if (num_temporal_layers_ == 3) { - gof.SetGofInfoVP9(kTemporalStructureMode3); - } - - // Encode at least (num_frames_in_gof + 1) frames to verify references - // of non-key frame with gof_idx = 0. - for (size_t frame_num = 0; frame_num < gof.num_frames_in_gof + 1; - ++frame_num) { - SetWaitForEncodedFramesThreshold(num_spatial_layers_); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, - encoder_->Encode(NextInputFrame(), nullptr)); - - const bool is_key_frame = frame_num == 0; - const size_t gof_idx = frame_num % gof.num_frames_in_gof; - const std::vector<uint8_t> p_diff(std::begin(gof.pid_diff[gof_idx]), - std::end(gof.pid_diff[gof_idx])); - - ExpectFrameWith(num_spatial_layers_, gof.temporal_idx[gof_idx], - gof.temporal_up_switch[gof_idx], - is_key_frame ? 0 : gof.num_ref_pics[gof_idx], p_diff); + int num_input_frames = gof.num_frames_in_gof + 1; + std::vector<EncodedVideoFrameProducer::EncodedFrame> frames = + EncodedVideoFrameProducer(*encoder) + .SetNumInputFrames(num_input_frames) + .SetResolution({kWidth, kHeight}) + .Encode(); + ASSERT_THAT(frames, SizeIs(num_input_frames * num_spatial_layers_)); + + for (size_t i = 0; i < frames.size(); ++i) { + const EncodedVideoFrameProducer::EncodedFrame& frame = frames[i]; + const size_t picture_idx = i / num_spatial_layers_; + const size_t gof_idx = picture_idx % gof.num_frames_in_gof; + + const CodecSpecificInfoVP9& vp9 = + frame.codec_specific_info.codecSpecific.VP9; + EXPECT_EQ(frame.encoded_image.SpatialIndex(), + num_spatial_layers_ == 1 + ? absl::nullopt + : absl::optional<int>(i % num_spatial_layers_)) + << "Frame " << i; + EXPECT_EQ(vp9.temporal_idx, num_temporal_layers_ == 1 + ? kNoTemporalIdx + : gof.temporal_idx[gof_idx]) + << "Frame " << i; + EXPECT_EQ(vp9.temporal_up_switch, gof.temporal_up_switch[gof_idx]) + << "Frame " << i; + if (picture_idx == 0) { + EXPECT_EQ(vp9.num_ref_pics, 0) << "Frame " << i; + } else { + EXPECT_THAT(rtc::MakeArrayView(vp9.p_diff, vp9.num_ref_pics), + UnorderedElementsAreArray(gof.pid_diff[gof_idx], + gof.num_ref_pics[gof_idx])) + << "Frame " << i; + } } } INSTANTIATE_TEST_SUITE_P(All, - TestVp9ImplWithLayering, + Vp9ImplWithLayeringTest, ::testing::Combine(::testing::Values(1, 2, 3), - ::testing::Values(1, 2, 3))); + ::testing::Values(1, 2, 3), + ::testing::Bool())); class TestVp9ImplFrameDropping : public TestVp9Impl { protected: @@ -1774,4 +1733,12 @@ TEST_F(TestVp9Impl, ReenablingUpperLayerAfterKFWithInterlayerPredIsEnabled) { EXPECT_EQ(encoded_frames[0]._frameType, VideoFrameType::kVideoFrameDelta); } +TEST_F(TestVp9Impl, HandlesEmptyInitDecode) { + std::unique_ptr<VideoDecoder> decoder = CreateDecoder(); + // Check that nullptr settings are ok for decoder. + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + decoder->InitDecode(/*codec_settings=*/nullptr, 1)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder->Release()); +} + } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9.cc index 527bce7729d..9b0585c0597 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9.cc @@ -39,6 +39,22 @@ std::vector<SdpVideoFormat> SupportedVP9Codecs() { cricket::kVp9CodecName, {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}})); } + + return supported_formats; +#else + return std::vector<SdpVideoFormat>(); +#endif +} + +std::vector<SdpVideoFormat> SupportedVP9DecoderCodecs() { +#ifdef RTC_ENABLE_VP9 + std::vector<SdpVideoFormat> supported_formats = SupportedVP9Codecs(); + // The WebRTC internal decoder supports VP9 profile 1. However, there's + // currently no way of sending VP9 profile 1 using the internal encoder. + // It would require extended support for I444, I422, and I440 buffers. + supported_formats.push_back(SdpVideoFormat( + cricket::kVp9CodecName, + {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile1)}})); return supported_formats; #else return std::vector<SdpVideoFormat>(); diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 028d3ab8f77..d29c19dc8c9 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -25,6 +25,7 @@ #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" +#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/checks.h" #include "rtc_base/experiments/rate_control_settings.h" #include "rtc_base/keep_ref_until_done.h" @@ -45,8 +46,6 @@ namespace { uint8_t kRefBufIdx[4] = {0, 0, 0, 1}; uint8_t kUpdBufIdx[4] = {0, 0, 1, 0}; -int kMaxNumTiles4kVideo = 8; - // Maximum allowed PID difference for differnet per-layer frame-rate case. const int kMaxAllowedPidDiff = 30; @@ -527,6 +526,11 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, config_->g_profile = 0; config_->g_input_bit_depth = 8; break; + case VP9Profile::kProfile1: + // Encoding of profile 1 is not implemented. It would require extended + // support for I444, I422, and I440 buffers. + RTC_NOTREACHED(); + break; case VP9Profile::kProfile2: img_fmt = VPX_IMG_FMT_I42016; bits_for_storage = 16; @@ -603,13 +607,6 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, (num_spatial_layers_ > 1 && codec_.mode == VideoCodecMode::kScreensharing) || inter_layer_pred_ == InterLayerPredMode::kOn; - // TODO(ilnik): Remove this workaround once external reference control works - // nicely with simulcast SVC mode. - // Simlucast SVC mode is currently only used in some tests and is impossible - // to trigger for users without using some field trials. - if (inter_layer_pred_ == InterLayerPredMode::kOff) { - external_ref_control_ = false; - } if (num_temporal_layers_ == 1) { gof_.SetGofInfoVP9(kTemporalStructureMode1); @@ -997,6 +994,10 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV(); break; } + case VP9Profile::kProfile1: { + RTC_NOTREACHED(); + break; + } case VP9Profile::kProfile2: { // We can inject kI010 frames directly for encode. All other formats // should be converted to it. @@ -1705,14 +1706,32 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) { // errors earlier than the multi-threads version. // - Make peak CPU usage under control (not depending on input) cfg.threads = 1; - (void)kMaxNumTiles4kVideo; // unused #else - // We want to use multithreading when decoding high resolution videos. But, - // since we don't know resolution of input stream at this stage, we always - // enable it. - cfg.threads = std::min(number_of_cores, kMaxNumTiles4kVideo); + if (!inst) { + // No config provided - don't know resolution to decode yet. + // Set thread count to one in the meantime. + cfg.threads = 1; + } else { + // We want to use multithreading when decoding high resolution videos. But + // not too many in order to avoid overhead when many stream are decoded + // concurrently. + // Set 2 thread as target for 1280x720 pixel count, and then scale up + // linearly from there - but cap at physical core count. + // For common resolutions this results in: + // 1 for 360p + // 2 for 720p + // 4 for 1080p + // 8 for 1440p + // 18 for 4K + int num_threads = + std::max(1, 2 * (inst->width * inst->height) / (1280 * 720)); + cfg.threads = std::min(number_of_cores, num_threads); + current_codec_ = *inst; + } #endif + num_cores_ = number_of_cores; + vpx_codec_flags_t flags = 0; if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) { return WEBRTC_VIDEO_CODEC_MEMORY; @@ -1730,6 +1749,15 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } } + + vpx_codec_err_t status = + vpx_codec_control(decoder_, VP9D_SET_LOOP_FILTER_OPT, 1); + if (status != VPX_CODEC_OK) { + RTC_LOG(LS_ERROR) << "Failed to enable VP9D_SET_LOOP_FILTER_OPT. " + << vpx_codec_error(decoder_); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + return WEBRTC_VIDEO_CODEC_OK; } @@ -1742,6 +1770,29 @@ int VP9DecoderImpl::Decode(const EncodedImage& input_image, if (decode_complete_callback_ == nullptr) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } + + if (input_image._frameType == VideoFrameType::kVideoFrameKey) { + absl::optional<vp9::FrameInfo> frame_info = + vp9::ParseIntraFrameInfo(input_image.data(), input_image.size()); + if (frame_info) { + if (frame_info->frame_width != current_codec_.width || + frame_info->frame_height != current_codec_.height) { + // Resolution has changed, tear down and re-init a new decoder in + // order to get correct sizing. + Release(); + current_codec_.width = frame_info->frame_width; + current_codec_.height = frame_info->frame_height; + int reinit_status = InitDecode(¤t_codec_, num_cores_); + if (reinit_status != WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_WARNING) << "Failed to re-init decoder."; + return reinit_status; + } + } + } else { + RTC_LOG(LS_WARNING) << "Failed to parse VP9 header from key-frame."; + } + } + // Always start with a complete key frame. if (key_frame_required_) { if (input_image._frameType != VideoFrameType::kVideoFrameKey) diff --git a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index f6d8318d7d5..fae94c752b9 100644 --- a/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/chromium/third_party/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -219,6 +219,8 @@ class VP9DecoderImpl : public VP9Decoder { bool inited_; vpx_codec_ctx_t* decoder_; bool key_frame_required_; + VideoCodec current_codec_; + int num_cores_; }; } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/deprecated/BUILD.gn b/chromium/third_party/webrtc/modules/video_coding/deprecated/BUILD.gn new file mode 100644 index 00000000000..f333b3f5e23 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/deprecated/BUILD.gn @@ -0,0 +1,33 @@ +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("../../../webrtc.gni") + +rtc_library("nack_module") { + sources = [ + "nack_module.cc", + "nack_module.h", + ] + + deps = [ + "..:nack_module", + "../..:module_api", + "../../../api/units:time_delta", + "../../../api/units:timestamp", + "../../../rtc_base:checks", + "../../../rtc_base:criticalsection", + "../../../rtc_base:deprecation", + "../../../rtc_base:logging", + "../../../rtc_base:macromagic", + "../../../rtc_base:rtc_numerics", + "../../../rtc_base/experiments:field_trial_parser", + "../../../system_wrappers", + "../../../system_wrappers:field_trial", + "../../utility", + ] +} diff --git a/chromium/third_party/webrtc/modules/video_coding/nack_module.cc b/chromium/third_party/webrtc/modules/video_coding/deprecated/nack_module.cc index 838af1548b0..8658729e99b 100644 --- a/chromium/third_party/webrtc/modules/video_coding/nack_module.cc +++ b/chromium/third_party/webrtc/modules/video_coding/deprecated/nack_module.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/nack_module.h" +#include "modules/video_coding/deprecated/nack_module.h" #include <algorithm> #include <limits> @@ -45,25 +45,25 @@ int64_t GetSendNackDelay() { } } // namespace -NackModule::NackInfo::NackInfo() +DEPRECATED_NackModule::NackInfo::NackInfo() : seq_num(0), send_at_seq_num(0), sent_at_time(-1), retries(0) {} -NackModule::NackInfo::NackInfo(uint16_t seq_num, - uint16_t send_at_seq_num, - int64_t created_at_time) +DEPRECATED_NackModule::NackInfo::NackInfo(uint16_t seq_num, + uint16_t send_at_seq_num, + int64_t created_at_time) : seq_num(seq_num), send_at_seq_num(send_at_seq_num), created_at_time(created_at_time), sent_at_time(-1), retries(0) {} -NackModule::BackoffSettings::BackoffSettings(TimeDelta min_retry, - TimeDelta max_rtt, - double base) +DEPRECATED_NackModule::BackoffSettings::BackoffSettings(TimeDelta min_retry, + TimeDelta max_rtt, + double base) : min_retry_interval(min_retry), max_rtt(max_rtt), base(base) {} -absl::optional<NackModule::BackoffSettings> -NackModule::BackoffSettings::ParseFromFieldTrials() { +absl::optional<DEPRECATED_NackModule::BackoffSettings> +DEPRECATED_NackModule::BackoffSettings::ParseFromFieldTrials() { // Matches magic number in RTPSender::OnReceivedNack(). const TimeDelta kDefaultMinRetryInterval = TimeDelta::Millis(5); // Upper bound on link-delay considered for exponential backoff. @@ -82,15 +82,16 @@ NackModule::BackoffSettings::ParseFromFieldTrials() { field_trial::FindFullName("WebRTC-ExponentialNackBackoff")); if (enabled) { - return NackModule::BackoffSettings(min_retry.Get(), max_rtt.Get(), - base.Get()); + return DEPRECATED_NackModule::BackoffSettings(min_retry.Get(), + max_rtt.Get(), base.Get()); } return absl::nullopt; } -NackModule::NackModule(Clock* clock, - NackSender* nack_sender, - KeyFrameRequestSender* keyframe_request_sender) +DEPRECATED_NackModule::DEPRECATED_NackModule( + Clock* clock, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender) : clock_(clock), nack_sender_(nack_sender), keyframe_request_sender_(keyframe_request_sender), @@ -106,13 +107,14 @@ NackModule::NackModule(Clock* clock, RTC_DCHECK(keyframe_request_sender_); } -int NackModule::OnReceivedPacket(uint16_t seq_num, bool is_keyframe) { +int DEPRECATED_NackModule::OnReceivedPacket(uint16_t seq_num, + bool is_keyframe) { return OnReceivedPacket(seq_num, is_keyframe, false); } -int NackModule::OnReceivedPacket(uint16_t seq_num, - bool is_keyframe, - bool is_recovered) { +int DEPRECATED_NackModule::OnReceivedPacket(uint16_t seq_num, + bool is_keyframe, + bool is_recovered) { rtc::CritScope lock(&crit_); // TODO(philipel): When the packet includes information whether it is // retransmitted or not, use that value instead. For @@ -181,7 +183,7 @@ int NackModule::OnReceivedPacket(uint16_t seq_num, return 0; } -void NackModule::ClearUpTo(uint16_t seq_num) { +void DEPRECATED_NackModule::ClearUpTo(uint16_t seq_num) { rtc::CritScope lock(&crit_); nack_list_.erase(nack_list_.begin(), nack_list_.lower_bound(seq_num)); keyframe_list_.erase(keyframe_list_.begin(), @@ -190,24 +192,24 @@ void NackModule::ClearUpTo(uint16_t seq_num) { recovered_list_.lower_bound(seq_num)); } -void NackModule::UpdateRtt(int64_t rtt_ms) { +void DEPRECATED_NackModule::UpdateRtt(int64_t rtt_ms) { rtc::CritScope lock(&crit_); rtt_ms_ = rtt_ms; } -void NackModule::Clear() { +void DEPRECATED_NackModule::Clear() { rtc::CritScope lock(&crit_); nack_list_.clear(); keyframe_list_.clear(); recovered_list_.clear(); } -int64_t NackModule::TimeUntilNextProcess() { +int64_t DEPRECATED_NackModule::TimeUntilNextProcess() { return std::max<int64_t>(next_process_time_ms_ - clock_->TimeInMilliseconds(), 0); } -void NackModule::Process() { +void DEPRECATED_NackModule::Process() { if (nack_sender_) { std::vector<uint16_t> nack_batch; { @@ -236,7 +238,7 @@ void NackModule::Process() { } } -bool NackModule::RemovePacketsUntilKeyFrame() { +bool DEPRECATED_NackModule::RemovePacketsUntilKeyFrame() { while (!keyframe_list_.empty()) { auto it = nack_list_.lower_bound(*keyframe_list_.begin()); @@ -254,8 +256,8 @@ bool NackModule::RemovePacketsUntilKeyFrame() { return false; } -void NackModule::AddPacketsToNack(uint16_t seq_num_start, - uint16_t seq_num_end) { +void DEPRECATED_NackModule::AddPacketsToNack(uint16_t seq_num_start, + uint16_t seq_num_end) { // Remove old packets. auto it = nack_list_.lower_bound(seq_num_end - kMaxPacketAge); nack_list_.erase(nack_list_.begin(), it); @@ -289,7 +291,8 @@ void NackModule::AddPacketsToNack(uint16_t seq_num_start, } } -std::vector<uint16_t> NackModule::GetNackBatch(NackFilterOptions options) { +std::vector<uint16_t> DEPRECATED_NackModule::GetNackBatch( + NackFilterOptions options) { bool consider_seq_num = options != kTimeOnly; bool consider_timestamp = options != kSeqNumOnly; Timestamp now = clock_->CurrentTime(); @@ -334,13 +337,13 @@ std::vector<uint16_t> NackModule::GetNackBatch(NackFilterOptions options) { return nack_batch; } -void NackModule::UpdateReorderingStatistics(uint16_t seq_num) { +void DEPRECATED_NackModule::UpdateReorderingStatistics(uint16_t seq_num) { RTC_DCHECK(AheadOf(newest_seq_num_, seq_num)); uint16_t diff = ReverseDiff(newest_seq_num_, seq_num); reordering_histogram_.Add(diff); } -int NackModule::WaitNumberOfPackets(float probability) const { +int DEPRECATED_NackModule::WaitNumberOfPackets(float probability) const { if (reordering_histogram_.NumValues() == 0) return 0; return reordering_histogram_.InverseCdf(probability); diff --git a/chromium/third_party/webrtc/modules/video_coding/nack_module.h b/chromium/third_party/webrtc/modules/video_coding/deprecated/nack_module.h index d4f705b3510..d704a05c11d 100644 --- a/chromium/third_party/webrtc/modules/video_coding/nack_module.h +++ b/chromium/third_party/webrtc/modules/video_coding/deprecated/nack_module.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef MODULES_VIDEO_CODING_NACK_MODULE_H_ -#define MODULES_VIDEO_CODING_NACK_MODULE_H_ +#ifndef MODULES_VIDEO_CODING_DEPRECATED_NACK_MODULE_H_ +#define MODULES_VIDEO_CODING_DEPRECATED_NACK_MODULE_H_ #include <stdint.h> @@ -22,17 +22,18 @@ #include "modules/include/module_common_types.h" #include "modules/video_coding/histogram.h" #include "rtc_base/critical_section.h" +#include "rtc_base/deprecation.h" #include "rtc_base/numerics/sequence_number_util.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" namespace webrtc { -class NackModule : public Module { +class DEPRECATED_NackModule : public Module { public: - NackModule(Clock* clock, - NackSender* nack_sender, - KeyFrameRequestSender* keyframe_request_sender); + DEPRECATED_NackModule(Clock* clock, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender); int OnReceivedPacket(uint16_t seq_num, bool is_keyframe); int OnReceivedPacket(uint16_t seq_num, bool is_keyframe, bool is_recovered); @@ -124,6 +125,8 @@ class NackModule : public Module { const absl::optional<BackoffSettings> backoff_settings_; }; +using NackModule = RTC_DEPRECATED DEPRECATED_NackModule; + } // namespace webrtc -#endif // MODULES_VIDEO_CODING_NACK_MODULE_H_ +#endif // MODULES_VIDEO_CODING_DEPRECATED_NACK_MODULE_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/encoded_frame.cc b/chromium/third_party/webrtc/modules/video_coding/encoded_frame.cc index 1e9e374c644..3de62da9f5e 100644 --- a/chromium/third_party/webrtc/modules/video_coding/encoded_frame.cc +++ b/chromium/third_party/webrtc/modules/video_coding/encoded_frame.cc @@ -135,20 +135,6 @@ void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header) { } case kVideoCodecH264: { _codecSpecificInfo.codecType = kVideoCodecH264; - - // The following H264 codec specific data are not used elsewhere. - // Instead they are read directly from the frame marking extension. - // These codec specific data structures should be removed - // when frame marking is used. - _codecSpecificInfo.codecSpecific.H264.temporal_idx = kNoTemporalIdx; - if (header->frame_marking.temporal_id != kNoTemporalIdx) { - _codecSpecificInfo.codecSpecific.H264.temporal_idx = - header->frame_marking.temporal_id; - _codecSpecificInfo.codecSpecific.H264.base_layer_sync = - header->frame_marking.base_layer_sync; - _codecSpecificInfo.codecSpecific.H264.idr_frame = - header->frame_marking.independent_frame; - } break; } default: { diff --git a/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.cc b/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.cc index 944f97bf871..64d3699e013 100644 --- a/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.cc +++ b/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.cc @@ -63,19 +63,25 @@ FrameBuffer::FrameBuffer(Clock* clock, last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs), add_rtt_to_playout_delay_( webrtc::field_trial::IsEnabled("WebRTC-AddRttToPlayoutDelay")), - rtt_mult_settings_(RttMultExperiment::GetRttMultValue()) {} + rtt_mult_settings_(RttMultExperiment::GetRttMultValue()) { + callback_checker_.Detach(); +} -FrameBuffer::~FrameBuffer() {} +FrameBuffer::~FrameBuffer() { + RTC_DCHECK_RUN_ON(&construction_checker_); +} void FrameBuffer::NextFrame( int64_t max_wait_time_ms, bool keyframe_required, rtc::TaskQueue* callback_queue, std::function<void(std::unique_ptr<EncodedFrame>, ReturnReason)> handler) { - RTC_DCHECK_RUN_ON(callback_queue); + RTC_DCHECK_RUN_ON(&callback_checker_); + RTC_DCHECK(callback_queue->IsCurrent()); TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); int64_t latest_return_time_ms = clock_->TimeInMilliseconds() + max_wait_time_ms; + rtc::CritScope lock(&crit_); if (stopped_) { return; @@ -93,6 +99,7 @@ void FrameBuffer::StartWaitForNextFrameOnQueue() { int64_t wait_ms = FindNextFrame(clock_->TimeInMilliseconds()); callback_task_ = RepeatingTaskHandle::DelayedStart( callback_queue_->Get(), TimeDelta::Millis(wait_ms), [this] { + RTC_DCHECK_RUN_ON(&callback_checker_); // If this task has not been cancelled, we did not get any new frames // while waiting. Continue with frame delivery. rtc::CritScope lock(&crit_); @@ -211,6 +218,7 @@ int64_t FrameBuffer::FindNextFrame(int64_t now_ms) { } EncodedFrame* FrameBuffer::GetNextFrame() { + RTC_DCHECK_RUN_ON(&callback_checker_); int64_t now_ms = clock_->TimeInMilliseconds(); // TODO(ilnik): remove |frames_out| use frames_to_decode_ directly. std::vector<EncodedFrame*> frames_out; @@ -334,7 +342,10 @@ void FrameBuffer::Start() { void FrameBuffer::Stop() { TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); rtc::CritScope lock(&crit_); + if (stopped_) + return; stopped_ = true; + CancelCallback(); } @@ -366,9 +377,11 @@ bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { } void FrameBuffer::CancelCallback() { + // Called from the callback queue or from within Stop(). frame_handler_ = {}; callback_task_.Stop(); callback_queue_ = nullptr; + callback_checker_.Detach(); } bool FrameBuffer::IsCompleteSuperFrame(const EncodedFrame& frame) { diff --git a/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.h b/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.h index 51f3820d317..d824ddf4d02 100644 --- a/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.h +++ b/chromium/third_party/webrtc/modules/video_coding/frame_buffer2.h @@ -28,6 +28,7 @@ #include "rtc_base/event.h" #include "rtc_base/experiments/rtt_mult_experiment.h" #include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/synchronization/sequence_checker.h" #include "rtc_base/task_queue.h" #include "rtc_base/task_utils/repeating_task.h" #include "rtc_base/thread_annotations.h" @@ -159,6 +160,9 @@ class FrameBuffer { EncodedFrame* CombineAndDeleteFrames( const std::vector<EncodedFrame*>& frames) const; + SequenceChecker construction_checker_; + SequenceChecker callback_checker_; + // Stores only undecoded frames. FrameMap frames_ RTC_GUARDED_BY(crit_); DecodedFramesHistory decoded_frames_history_ RTC_GUARDED_BY(crit_); diff --git a/chromium/third_party/webrtc/modules/video_coding/frame_buffer2_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/frame_buffer2_unittest.cc index b4d663ee063..2de3f3362b0 100644 --- a/chromium/third_party/webrtc/modules/video_coding/frame_buffer2_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/frame_buffer2_unittest.cc @@ -108,21 +108,26 @@ class FrameObjectFake : public EncodedFrame { class VCMReceiveStatisticsCallbackMock : public VCMReceiveStatisticsCallback { public: - MOCK_METHOD3(OnCompleteFrame, - void(bool is_keyframe, - size_t size_bytes, - VideoContentType content_type)); - MOCK_METHOD1(OnDroppedFrames, void(uint32_t frames_dropped)); - MOCK_METHOD1(OnDiscardedPacketsUpdated, void(int discarded_packets)); - MOCK_METHOD1(OnFrameCountsUpdated, void(const FrameCounts& frame_counts)); - MOCK_METHOD6(OnFrameBufferTimingsUpdated, - void(int max_decode_ms, - int current_delay_ms, - int target_delay_ms, - int jitter_buffer_ms, - int min_playout_delay_ms, - int render_delay_ms)); - MOCK_METHOD1(OnTimingFrameInfoUpdated, void(const TimingFrameInfo& info)); + MOCK_METHOD(void, + OnCompleteFrame, + (bool is_keyframe, + size_t size_bytes, + VideoContentType content_type), + (override)); + MOCK_METHOD(void, OnDroppedFrames, (uint32_t frames_dropped), (override)); + MOCK_METHOD(void, + OnFrameBufferTimingsUpdated, + (int max_decode_ms, + int current_delay_ms, + int target_delay_ms, + int jitter_buffer_ms, + int min_playout_delay_ms, + int render_delay_ms), + (override)); + MOCK_METHOD(void, + OnTimingFrameInfoUpdated, + (const TimingFrameInfo& info), + (override)); }; class TestFrameBuffer2 : public ::testing::Test { diff --git a/chromium/third_party/webrtc/modules/video_coding/frame_object.cc b/chromium/third_party/webrtc/modules/video_coding/frame_object.cc index cb83999c942..2399e8f060f 100644 --- a/chromium/third_party/webrtc/modules/video_coding/frame_object.cc +++ b/chromium/third_party/webrtc/modules/video_coding/frame_object.cc @@ -129,9 +129,5 @@ const RTPVideoHeader& RtpFrameObject::GetRtpVideoHeader() const { return rtp_video_header_; } -const FrameMarking& RtpFrameObject::GetFrameMarking() const { - return rtp_video_header_.frame_marking; -} - } // namespace video_coding } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/frame_object.h b/chromium/third_party/webrtc/modules/video_coding/frame_object.h index f7988763d38..831b444df42 100644 --- a/chromium/third_party/webrtc/modules/video_coding/frame_object.h +++ b/chromium/third_party/webrtc/modules/video_coding/frame_object.h @@ -47,7 +47,6 @@ class RtpFrameObject : public EncodedFrame { int64_t RenderTime() const override; bool delayed_by_retransmission() const override; const RTPVideoHeader& GetRtpVideoHeader() const; - const FrameMarking& GetFrameMarking() const; private: RTPVideoHeader rtp_video_header_; diff --git a/chromium/third_party/webrtc/modules/video_coding/h264_sps_pps_tracker.cc b/chromium/third_party/webrtc/modules/video_coding/h264_sps_pps_tracker.cc index 3965b28e8ec..4becdb76085 100644 --- a/chromium/third_party/webrtc/modules/video_coding/h264_sps_pps_tracker.cc +++ b/chromium/third_party/webrtc/modules/video_coding/h264_sps_pps_tracker.cc @@ -49,6 +49,7 @@ H264SpsPpsTracker::FixedBitstream H264SpsPpsTracker::CopyAndFixBitstream( RTPVideoHeader* video_header) { RTC_DCHECK(video_header); RTC_DCHECK(video_header->codec == kVideoCodecH264); + RTC_DCHECK_GT(bitstream.size(), 0); auto& h264_header = absl::get<RTPVideoHeaderH264>(video_header->video_type_header); @@ -128,7 +129,7 @@ H264SpsPpsTracker::FixedBitstream H264SpsPpsTracker::CopyAndFixBitstream( if (h264_header.packetization_type == kH264StapA) { const uint8_t* nalu_ptr = bitstream.data() + 1; - while (nalu_ptr < bitstream.data() + bitstream.size()) { + while (nalu_ptr < bitstream.data() + bitstream.size() - 1) { RTC_DCHECK(video_header->is_first_packet_in_frame); required_size += sizeof(start_code_h264); @@ -180,7 +181,7 @@ H264SpsPpsTracker::FixedBitstream H264SpsPpsTracker::CopyAndFixBitstream( // Copy the rest of the bitstream and insert start codes. if (h264_header.packetization_type == kH264StapA) { const uint8_t* nalu_ptr = bitstream.data() + 1; - while (nalu_ptr < bitstream.data() + bitstream.size()) { + while (nalu_ptr < bitstream.data() + bitstream.size() - 1) { fixed.bitstream.AppendData(start_code_h264); // The first two bytes describe the length of a segment. diff --git a/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.cc b/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.cc index cd505835d11..44e2a9811e7 100644 --- a/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.cc +++ b/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.cc @@ -23,6 +23,7 @@ #include "rtc_base/experiments/jitter_upper_bound_experiment.h" #include "rtc_base/numerics/safe_conversions.h" #include "system_wrappers/include/clock.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { namespace { @@ -50,6 +51,8 @@ VCMJitterEstimator::VCMJitterEstimator(Clock* clock) time_deviation_upper_bound_( JitterUpperBoundExperiment::GetUpperBoundSigmas().value_or( kDefaultMaxTimestampDeviationInSigmas)), + enable_reduced_delay_( + !field_trial::IsEnabled("WebRTC-ReducedJitterDelayKillSwitch")), clock_(clock) { Reset(); } @@ -395,22 +398,25 @@ int VCMJitterEstimator::GetJitterEstimate( } } - static const double kJitterScaleLowThreshold = 5.0; - static const double kJitterScaleHighThreshold = 10.0; - double fps = GetFrameRate(); - // Ignore jitter for very low fps streams. - if (fps < kJitterScaleLowThreshold) { - if (fps == 0.0) { - return rtc::checked_cast<int>(std::max(0.0, jitterMS) + 0.5); + if (enable_reduced_delay_) { + static const double kJitterScaleLowThreshold = 5.0; + static const double kJitterScaleHighThreshold = 10.0; + double fps = GetFrameRate(); + // Ignore jitter for very low fps streams. + if (fps < kJitterScaleLowThreshold) { + if (fps == 0.0) { + return rtc::checked_cast<int>(std::max(0.0, jitterMS) + 0.5); + } + return 0; } - return 0; - } - // Semi-low frame rate; scale by factor linearly interpolated from 0.0 at - // kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold. - if (fps < kJitterScaleHighThreshold) { - jitterMS = (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) * - (fps - kJitterScaleLowThreshold) * jitterMS; + // Semi-low frame rate; scale by factor linearly interpolated from 0.0 at + // kJitterScaleLowThreshold to 1.0 at kJitterScaleHighThreshold. + if (fps < kJitterScaleHighThreshold) { + jitterMS = + (1.0 / (kJitterScaleHighThreshold - kJitterScaleLowThreshold)) * + (fps - kJitterScaleLowThreshold) * jitterMS; + } } return rtc::checked_cast<int>(std::max(0.0, jitterMS) + 0.5); diff --git a/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.h b/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.h index d9798b40a19..1d69b957694 100644 --- a/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.h +++ b/chromium/third_party/webrtc/modules/video_coding/jitter_estimator.h @@ -150,6 +150,7 @@ class VCMJitterEstimator { rtc::RollingAccumulator<uint64_t> fps_counter_; const double time_deviation_upper_bound_; + const bool enable_reduced_delay_; Clock* clock_; }; diff --git a/chromium/third_party/webrtc/modules/video_coding/jitter_estimator_tests.cc b/chromium/third_party/webrtc/modules/video_coding/jitter_estimator_tests.cc index 1ad9abb56f6..14baae7e816 100644 --- a/chromium/third_party/webrtc/modules/video_coding/jitter_estimator_tests.cc +++ b/chromium/third_party/webrtc/modules/video_coding/jitter_estimator_tests.cc @@ -72,6 +72,22 @@ TEST_F(TestVCMJitterEstimator, TestLowRate) { } } +TEST_F(TestVCMJitterEstimator, TestLowRateDisabled) { + test::ScopedFieldTrials field_trials( + "WebRTC-ReducedJitterDelayKillSwitch/Enabled/"); + SetUp(); + + ValueGenerator gen(10); + uint64_t time_delta_us = rtc::kNumMicrosecsPerSec / 5; + for (int i = 0; i < 60; ++i) { + estimator_->UpdateEstimate(gen.Delay(), gen.FrameSize()); + AdvanceClock(time_delta_us); + if (i > 2) + EXPECT_GT(estimator_->GetJitterEstimate(0, absl::nullopt), 0); + gen.Advance(); + } +} + TEST_F(TestVCMJitterEstimator, TestUpperBound) { struct TestContext { TestContext() diff --git a/chromium/third_party/webrtc/modules/video_coding/nack_module2.cc b/chromium/third_party/webrtc/modules/video_coding/nack_module2.cc new file mode 100644 index 00000000000..8a3a731ed0c --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/nack_module2.cc @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/nack_module2.h" + +#include <algorithm> +#include <limits> + +#include "api/units/timestamp.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/field_trial_parser.h" +#include "rtc_base/logging.h" +#include "rtc_base/task_queue.h" +#include "system_wrappers/include/field_trial.h" + +namespace webrtc { + +namespace { +const int kMaxPacketAge = 10000; +const int kMaxNackPackets = 1000; +const int kDefaultRttMs = 100; +const int kMaxNackRetries = 10; +const int kMaxReorderedPackets = 128; +const int kNumReorderingBuckets = 10; +const int kDefaultSendNackDelayMs = 0; + +int64_t GetSendNackDelay() { + int64_t delay_ms = strtol( + webrtc::field_trial::FindFullName("WebRTC-SendNackDelayMs").c_str(), + nullptr, 10); + if (delay_ms > 0 && delay_ms <= 20) { + RTC_LOG(LS_INFO) << "SendNackDelay is set to " << delay_ms; + return delay_ms; + } + return kDefaultSendNackDelayMs; +} +} // namespace + +constexpr TimeDelta NackModule2::kUpdateInterval; + +NackModule2::NackInfo::NackInfo() + : seq_num(0), send_at_seq_num(0), sent_at_time(-1), retries(0) {} + +NackModule2::NackInfo::NackInfo(uint16_t seq_num, + uint16_t send_at_seq_num, + int64_t created_at_time) + : seq_num(seq_num), + send_at_seq_num(send_at_seq_num), + created_at_time(created_at_time), + sent_at_time(-1), + retries(0) {} + +NackModule2::BackoffSettings::BackoffSettings(TimeDelta min_retry, + TimeDelta max_rtt, + double base) + : min_retry_interval(min_retry), max_rtt(max_rtt), base(base) {} + +absl::optional<NackModule2::BackoffSettings> +NackModule2::BackoffSettings::ParseFromFieldTrials() { + // Matches magic number in RTPSender::OnReceivedNack(). + const TimeDelta kDefaultMinRetryInterval = TimeDelta::Millis(5); + // Upper bound on link-delay considered for exponential backoff. + // Selected so that cumulative delay with 1.25 base and 10 retries ends up + // below 3s, since above that there will be a FIR generated instead. + const TimeDelta kDefaultMaxRtt = TimeDelta::Millis(160); + // Default base for exponential backoff, adds 25% RTT delay for each retry. + const double kDefaultBase = 1.25; + + FieldTrialParameter<bool> enabled("enabled", false); + FieldTrialParameter<TimeDelta> min_retry("min_retry", + kDefaultMinRetryInterval); + FieldTrialParameter<TimeDelta> max_rtt("max_rtt", kDefaultMaxRtt); + FieldTrialParameter<double> base("base", kDefaultBase); + ParseFieldTrial({&enabled, &min_retry, &max_rtt, &base}, + field_trial::FindFullName("WebRTC-ExponentialNackBackoff")); + + if (enabled) { + return NackModule2::BackoffSettings(min_retry.Get(), max_rtt.Get(), + base.Get()); + } + return absl::nullopt; +} + +NackModule2::NackModule2(TaskQueueBase* current_queue, + Clock* clock, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + TimeDelta update_interval /*= kUpdateInterval*/) + : worker_thread_(current_queue), + update_interval_(update_interval), + clock_(clock), + nack_sender_(nack_sender), + keyframe_request_sender_(keyframe_request_sender), + reordering_histogram_(kNumReorderingBuckets, kMaxReorderedPackets), + initialized_(false), + rtt_ms_(kDefaultRttMs), + newest_seq_num_(0), + send_nack_delay_ms_(GetSendNackDelay()), + backoff_settings_(BackoffSettings::ParseFromFieldTrials()) { + RTC_DCHECK(clock_); + RTC_DCHECK(nack_sender_); + RTC_DCHECK(keyframe_request_sender_); + RTC_DCHECK_GT(update_interval.ms(), 0); + RTC_DCHECK(worker_thread_); + RTC_DCHECK(worker_thread_->IsCurrent()); + + repeating_task_ = RepeatingTaskHandle::DelayedStart( + TaskQueueBase::Current(), update_interval_, + [this]() { + RTC_DCHECK_RUN_ON(worker_thread_); + std::vector<uint16_t> nack_batch = GetNackBatch(kTimeOnly); + if (!nack_batch.empty()) { + // This batch of NACKs is triggered externally; there is no external + // initiator who can batch them with other feedback messages. + nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/false); + } + return update_interval_; + }, + clock_); +} + +NackModule2::~NackModule2() { + RTC_DCHECK_RUN_ON(worker_thread_); + repeating_task_.Stop(); +} + +int NackModule2::OnReceivedPacket(uint16_t seq_num, bool is_keyframe) { + RTC_DCHECK_RUN_ON(worker_thread_); + return OnReceivedPacket(seq_num, is_keyframe, false); +} + +int NackModule2::OnReceivedPacket(uint16_t seq_num, + bool is_keyframe, + bool is_recovered) { + RTC_DCHECK_RUN_ON(worker_thread_); + // TODO(philipel): When the packet includes information whether it is + // retransmitted or not, use that value instead. For + // now set it to true, which will cause the reordering + // statistics to never be updated. + bool is_retransmitted = true; + + if (!initialized_) { + newest_seq_num_ = seq_num; + if (is_keyframe) + keyframe_list_.insert(seq_num); + initialized_ = true; + return 0; + } + + // Since the |newest_seq_num_| is a packet we have actually received we know + // that packet has never been Nacked. + if (seq_num == newest_seq_num_) + return 0; + + if (AheadOf(newest_seq_num_, seq_num)) { + // An out of order packet has been received. + auto nack_list_it = nack_list_.find(seq_num); + int nacks_sent_for_packet = 0; + if (nack_list_it != nack_list_.end()) { + nacks_sent_for_packet = nack_list_it->second.retries; + nack_list_.erase(nack_list_it); + } + if (!is_retransmitted) + UpdateReorderingStatistics(seq_num); + return nacks_sent_for_packet; + } + + // Keep track of new keyframes. + if (is_keyframe) + keyframe_list_.insert(seq_num); + + // And remove old ones so we don't accumulate keyframes. + auto it = keyframe_list_.lower_bound(seq_num - kMaxPacketAge); + if (it != keyframe_list_.begin()) + keyframe_list_.erase(keyframe_list_.begin(), it); + + if (is_recovered) { + recovered_list_.insert(seq_num); + + // Remove old ones so we don't accumulate recovered packets. + auto it = recovered_list_.lower_bound(seq_num - kMaxPacketAge); + if (it != recovered_list_.begin()) + recovered_list_.erase(recovered_list_.begin(), it); + + // Do not send nack for packets recovered by FEC or RTX. + return 0; + } + + AddPacketsToNack(newest_seq_num_ + 1, seq_num); + newest_seq_num_ = seq_num; + + // Are there any nacks that are waiting for this seq_num. + std::vector<uint16_t> nack_batch = GetNackBatch(kSeqNumOnly); + if (!nack_batch.empty()) { + // This batch of NACKs is triggered externally; the initiator can + // batch them with other feedback messages. + nack_sender_->SendNack(nack_batch, /*buffering_allowed=*/true); + } + + return 0; +} + +void NackModule2::ClearUpTo(uint16_t seq_num) { + // Called via RtpVideoStreamReceiver2::FrameContinuous on the network thread. + worker_thread_->PostTask(ToQueuedTask(task_safety_, [seq_num, this]() { + RTC_DCHECK_RUN_ON(worker_thread_); + nack_list_.erase(nack_list_.begin(), nack_list_.lower_bound(seq_num)); + keyframe_list_.erase(keyframe_list_.begin(), + keyframe_list_.lower_bound(seq_num)); + recovered_list_.erase(recovered_list_.begin(), + recovered_list_.lower_bound(seq_num)); + })); +} + +void NackModule2::UpdateRtt(int64_t rtt_ms) { + RTC_DCHECK_RUN_ON(worker_thread_); + rtt_ms_ = rtt_ms; +} + +bool NackModule2::RemovePacketsUntilKeyFrame() { + // Called on worker_thread_. + while (!keyframe_list_.empty()) { + auto it = nack_list_.lower_bound(*keyframe_list_.begin()); + + if (it != nack_list_.begin()) { + // We have found a keyframe that actually is newer than at least one + // packet in the nack list. + nack_list_.erase(nack_list_.begin(), it); + return true; + } + + // If this keyframe is so old it does not remove any packets from the list, + // remove it from the list of keyframes and try the next keyframe. + keyframe_list_.erase(keyframe_list_.begin()); + } + return false; +} + +void NackModule2::AddPacketsToNack(uint16_t seq_num_start, + uint16_t seq_num_end) { + // Called on worker_thread_. + // Remove old packets. + auto it = nack_list_.lower_bound(seq_num_end - kMaxPacketAge); + nack_list_.erase(nack_list_.begin(), it); + + // If the nack list is too large, remove packets from the nack list until + // the latest first packet of a keyframe. If the list is still too large, + // clear it and request a keyframe. + uint16_t num_new_nacks = ForwardDiff(seq_num_start, seq_num_end); + if (nack_list_.size() + num_new_nacks > kMaxNackPackets) { + while (RemovePacketsUntilKeyFrame() && + nack_list_.size() + num_new_nacks > kMaxNackPackets) { + } + + if (nack_list_.size() + num_new_nacks > kMaxNackPackets) { + nack_list_.clear(); + RTC_LOG(LS_WARNING) << "NACK list full, clearing NACK" + " list and requesting keyframe."; + keyframe_request_sender_->RequestKeyFrame(); + return; + } + } + + for (uint16_t seq_num = seq_num_start; seq_num != seq_num_end; ++seq_num) { + // Do not send nack for packets that are already recovered by FEC or RTX + if (recovered_list_.find(seq_num) != recovered_list_.end()) + continue; + NackInfo nack_info(seq_num, seq_num + WaitNumberOfPackets(0.5), + clock_->TimeInMilliseconds()); + RTC_DCHECK(nack_list_.find(seq_num) == nack_list_.end()); + nack_list_[seq_num] = nack_info; + } +} + +std::vector<uint16_t> NackModule2::GetNackBatch(NackFilterOptions options) { + // Called on worker_thread_. + + bool consider_seq_num = options != kTimeOnly; + bool consider_timestamp = options != kSeqNumOnly; + Timestamp now = clock_->CurrentTime(); + std::vector<uint16_t> nack_batch; + auto it = nack_list_.begin(); + while (it != nack_list_.end()) { + TimeDelta resend_delay = TimeDelta::Millis(rtt_ms_); + if (backoff_settings_) { + resend_delay = + std::max(resend_delay, backoff_settings_->min_retry_interval); + if (it->second.retries > 1) { + TimeDelta exponential_backoff = + std::min(TimeDelta::Millis(rtt_ms_), backoff_settings_->max_rtt) * + std::pow(backoff_settings_->base, it->second.retries - 1); + resend_delay = std::max(resend_delay, exponential_backoff); + } + } + + bool delay_timed_out = + now.ms() - it->second.created_at_time >= send_nack_delay_ms_; + bool nack_on_rtt_passed = + now.ms() - it->second.sent_at_time >= resend_delay.ms(); + bool nack_on_seq_num_passed = + it->second.sent_at_time == -1 && + AheadOrAt(newest_seq_num_, it->second.send_at_seq_num); + if (delay_timed_out && ((consider_seq_num && nack_on_seq_num_passed) || + (consider_timestamp && nack_on_rtt_passed))) { + nack_batch.emplace_back(it->second.seq_num); + ++it->second.retries; + it->second.sent_at_time = now.ms(); + if (it->second.retries >= kMaxNackRetries) { + RTC_LOG(LS_WARNING) << "Sequence number " << it->second.seq_num + << " removed from NACK list due to max retries."; + it = nack_list_.erase(it); + } else { + ++it; + } + continue; + } + ++it; + } + return nack_batch; +} + +void NackModule2::UpdateReorderingStatistics(uint16_t seq_num) { + // Running on worker_thread_. + RTC_DCHECK(AheadOf(newest_seq_num_, seq_num)); + uint16_t diff = ReverseDiff(newest_seq_num_, seq_num); + reordering_histogram_.Add(diff); +} + +int NackModule2::WaitNumberOfPackets(float probability) const { + // Called on worker_thread_; + if (reordering_histogram_.NumValues() == 0) + return 0; + return reordering_histogram_.InverseCdf(probability); +} + +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/nack_module2.h b/chromium/third_party/webrtc/modules/video_coding/nack_module2.h new file mode 100644 index 00000000000..89dd0821922 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/nack_module2.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_NACK_MODULE2_H_ +#define MODULES_VIDEO_CODING_NACK_MODULE2_H_ + +#include <stdint.h> + +#include <map> +#include <set> +#include <vector> + +#include "api/units/time_delta.h" +#include "modules/include/module_common_types.h" +#include "modules/video_coding/histogram.h" +#include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/task_utils/pending_task_safety_flag.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +// TODO(bugs.webrtc.org/11594): This class no longer implements the Module +// interface and therefore "NackModule" may not be a descriptive name anymore. +// Consider renaming to e.g. NackTracker or NackRequester. +class NackModule2 final { + public: + static constexpr TimeDelta kUpdateInterval = TimeDelta::Millis(20); + + NackModule2(TaskQueueBase* current_queue, + Clock* clock, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + TimeDelta update_interval = kUpdateInterval); + ~NackModule2(); + + int OnReceivedPacket(uint16_t seq_num, bool is_keyframe); + int OnReceivedPacket(uint16_t seq_num, bool is_keyframe, bool is_recovered); + + void ClearUpTo(uint16_t seq_num); + void UpdateRtt(int64_t rtt_ms); + + private: + // Which fields to consider when deciding which packet to nack in + // GetNackBatch. + enum NackFilterOptions { kSeqNumOnly, kTimeOnly, kSeqNumAndTime }; + + // This class holds the sequence number of the packet that is in the nack list + // as well as the meta data about when it should be nacked and how many times + // we have tried to nack this packet. + struct NackInfo { + NackInfo(); + NackInfo(uint16_t seq_num, + uint16_t send_at_seq_num, + int64_t created_at_time); + + uint16_t seq_num; + uint16_t send_at_seq_num; + int64_t created_at_time; + int64_t sent_at_time; + int retries; + }; + + struct BackoffSettings { + BackoffSettings(TimeDelta min_retry, TimeDelta max_rtt, double base); + static absl::optional<BackoffSettings> ParseFromFieldTrials(); + + // Min time between nacks. + const TimeDelta min_retry_interval; + // Upper bound on link-delay considered for exponential backoff. + const TimeDelta max_rtt; + // Base for the exponential backoff. + const double base; + }; + + void AddPacketsToNack(uint16_t seq_num_start, uint16_t seq_num_end) + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); + + // Removes packets from the nack list until the next keyframe. Returns true + // if packets were removed. + bool RemovePacketsUntilKeyFrame() + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); + std::vector<uint16_t> GetNackBatch(NackFilterOptions options) + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); + + // Update the reordering distribution. + void UpdateReorderingStatistics(uint16_t seq_num) + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); + + // Returns how many packets we have to wait in order to receive the packet + // with probability |probabilty| or higher. + int WaitNumberOfPackets(float probability) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(worker_thread_); + + TaskQueueBase* const worker_thread_; + + // Used to regularly call SendNack if needed. + RepeatingTaskHandle repeating_task_ RTC_GUARDED_BY(worker_thread_); + const TimeDelta update_interval_; + + Clock* const clock_; + NackSender* const nack_sender_; + KeyFrameRequestSender* const keyframe_request_sender_; + + // TODO(philipel): Some of the variables below are consistently used on a + // known thread (e.g. see |initialized_|). Those probably do not need + // synchronized access. + std::map<uint16_t, NackInfo, DescendingSeqNumComp<uint16_t>> nack_list_ + RTC_GUARDED_BY(worker_thread_); + std::set<uint16_t, DescendingSeqNumComp<uint16_t>> keyframe_list_ + RTC_GUARDED_BY(worker_thread_); + std::set<uint16_t, DescendingSeqNumComp<uint16_t>> recovered_list_ + RTC_GUARDED_BY(worker_thread_); + video_coding::Histogram reordering_histogram_ RTC_GUARDED_BY(worker_thread_); + bool initialized_ RTC_GUARDED_BY(worker_thread_); + int64_t rtt_ms_ RTC_GUARDED_BY(worker_thread_); + uint16_t newest_seq_num_ RTC_GUARDED_BY(worker_thread_); + + // Adds a delay before send nack on packet received. + const int64_t send_nack_delay_ms_; + + const absl::optional<BackoffSettings> backoff_settings_; + + // Used to signal destruction to potentially pending tasks. + ScopedTaskSafety task_safety_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_NACK_MODULE2_H_ diff --git a/chromium/third_party/webrtc/modules/video_coding/nack_module2_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/nack_module2_unittest.cc new file mode 100644 index 00000000000..acd1eead012 --- /dev/null +++ b/chromium/third_party/webrtc/modules/video_coding/nack_module2_unittest.cc @@ -0,0 +1,411 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/nack_module2.h" + +#include <algorithm> +#include <cstdint> +#include <cstring> +#include <memory> + +#include "system_wrappers/include/clock.h" +#include "test/field_trial.h" +#include "test/gtest.h" +#include "test/run_loop.h" + +namespace webrtc { +// TODO(bugs.webrtc.org/11594): Use the use the GlobalSimulatedTimeController +// instead of RunLoop. At the moment we mix use of the Clock and the underlying +// implementation of RunLoop, which is realtime. +class TestNackModule2 : public ::testing::TestWithParam<bool>, + public NackSender, + public KeyFrameRequestSender { + protected: + TestNackModule2() + : clock_(new SimulatedClock(0)), + field_trial_(GetParam() + ? "WebRTC-ExponentialNackBackoff/enabled:true/" + : "WebRTC-ExponentialNackBackoff/enabled:false/"), + keyframes_requested_(0) {} + + void SetUp() override {} + + void SendNack(const std::vector<uint16_t>& sequence_numbers, + bool buffering_allowed) override { + sent_nacks_.insert(sent_nacks_.end(), sequence_numbers.begin(), + sequence_numbers.end()); + if (waiting_for_send_nack_) { + waiting_for_send_nack_ = false; + loop_.Quit(); + } + } + + void RequestKeyFrame() override { ++keyframes_requested_; } + + void Flush() { + // nack_module.Process(); + loop_.Flush(); + } + + bool WaitForSendNack() { + if (timed_out_) { + RTC_NOTREACHED(); + return false; + } + + RTC_DCHECK(!waiting_for_send_nack_); + + waiting_for_send_nack_ = true; + loop_.PostDelayedTask( + [this]() { + timed_out_ = true; + loop_.Quit(); + }, + 1000); + + loop_.Run(); + + if (timed_out_) + return false; + + RTC_DCHECK(!waiting_for_send_nack_); + return true; + } + + NackModule2& CreateNackModule( + TimeDelta interval = NackModule2::kUpdateInterval) { + RTC_DCHECK(!nack_module_.get()); + nack_module_ = std::make_unique<NackModule2>( + TaskQueueBase::Current(), clock_.get(), this, this, interval); + nack_module_->UpdateRtt(kDefaultRttMs); + return *nack_module_.get(); + } + + static constexpr int64_t kDefaultRttMs = 20; + test::RunLoop loop_; + std::unique_ptr<SimulatedClock> clock_; + test::ScopedFieldTrials field_trial_; + std::unique_ptr<NackModule2> nack_module_; + std::vector<uint16_t> sent_nacks_; + int keyframes_requested_; + bool waiting_for_send_nack_ = false; + bool timed_out_ = false; +}; + +TEST_P(TestNackModule2, NackOnePacket) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(1, false, false); + nack_module.OnReceivedPacket(3, false, false); + ASSERT_EQ(1u, sent_nacks_.size()); + EXPECT_EQ(2, sent_nacks_[0]); +} + +TEST_P(TestNackModule2, WrappingSeqNum) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(0xfffe, false, false); + nack_module.OnReceivedPacket(1, false, false); + ASSERT_EQ(2u, sent_nacks_.size()); + EXPECT_EQ(0xffff, sent_nacks_[0]); + EXPECT_EQ(0, sent_nacks_[1]); +} + +TEST_P(TestNackModule2, WrappingSeqNumClearToKeyframe) { + NackModule2& nack_module = CreateNackModule(TimeDelta::Millis(10)); + nack_module.OnReceivedPacket(0xfffe, false, false); + nack_module.OnReceivedPacket(1, false, false); + ASSERT_EQ(2u, sent_nacks_.size()); + EXPECT_EQ(0xffff, sent_nacks_[0]); + EXPECT_EQ(0, sent_nacks_[1]); + + sent_nacks_.clear(); + nack_module.OnReceivedPacket(2, true, false); + ASSERT_EQ(0u, sent_nacks_.size()); + + nack_module.OnReceivedPacket(501, true, false); + ASSERT_EQ(498u, sent_nacks_.size()); + for (int seq_num = 3; seq_num < 501; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]); + + sent_nacks_.clear(); + nack_module.OnReceivedPacket(1001, false, false); + EXPECT_EQ(499u, sent_nacks_.size()); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + ASSERT_TRUE(WaitForSendNack()); + ASSERT_EQ(999u, sent_nacks_.size()); + EXPECT_EQ(0xffff, sent_nacks_[0]); + EXPECT_EQ(0, sent_nacks_[1]); + for (int seq_num = 3; seq_num < 501; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 1]); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 2]); + + // Adding packet 1004 will cause the nack list to reach it's max limit. + // It will then clear all nacks up to the next keyframe (seq num 2), + // thus removing 0xffff and 0 from the nack list. + sent_nacks_.clear(); + nack_module.OnReceivedPacket(1004, false, false); + ASSERT_EQ(2u, sent_nacks_.size()); + EXPECT_EQ(1002, sent_nacks_[0]); + EXPECT_EQ(1003, sent_nacks_[1]); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + ASSERT_TRUE(WaitForSendNack()); + ASSERT_EQ(999u, sent_nacks_.size()); + for (int seq_num = 3; seq_num < 501; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 3]); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 4]); + + // Adding packet 1007 will cause the nack module to overflow again, thus + // clearing everything up to 501 which is the next keyframe. + nack_module.OnReceivedPacket(1007, false, false); + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + ASSERT_TRUE(WaitForSendNack()); + ASSERT_EQ(503u, sent_nacks_.size()); + for (int seq_num = 502; seq_num < 1001; ++seq_num) + EXPECT_EQ(seq_num, sent_nacks_[seq_num - 502]); + EXPECT_EQ(1005, sent_nacks_[501]); + EXPECT_EQ(1006, sent_nacks_[502]); +} + +TEST_P(TestNackModule2, ResendNack) { + NackModule2& nack_module = CreateNackModule(TimeDelta::Millis(1)); + nack_module.OnReceivedPacket(1, false, false); + nack_module.OnReceivedPacket(3, false, false); + size_t expected_nacks_sent = 1; + ASSERT_EQ(expected_nacks_sent, sent_nacks_.size()); + EXPECT_EQ(2, sent_nacks_[0]); + + if (GetParam()) { + // Retry has to wait at least 5ms by default. + nack_module.UpdateRtt(1); + clock_->AdvanceTimeMilliseconds(4); + Flush(); // Too early. + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); + + clock_->AdvanceTimeMilliseconds(1); + WaitForSendNack(); // Now allowed. + EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size()); + } else { + nack_module.UpdateRtt(1); + clock_->AdvanceTimeMilliseconds(1); + WaitForSendNack(); // Fast retransmit allowed. + EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size()); + } + + // N:th try has to wait b^(N-1) * rtt by default. + const double b = GetParam() ? 1.25 : 1.0; + for (int i = 2; i < 10; ++i) { + // Change RTT, above the 40ms max for exponential backoff. + TimeDelta rtt = TimeDelta::Millis(160); // + (i * 10 - 40) + nack_module.UpdateRtt(rtt.ms()); + + // RTT gets capped at 160ms in backoff calculations. + TimeDelta expected_backoff_delay = + std::pow(b, i - 1) * std::min(rtt, TimeDelta::Millis(160)); + + // Move to one millisecond before next allowed NACK. + clock_->AdvanceTimeMilliseconds(expected_backoff_delay.ms() - 1); + Flush(); + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); + + // Move to one millisecond after next allowed NACK. + // After rather than on to avoid rounding errors. + clock_->AdvanceTimeMilliseconds(2); + WaitForSendNack(); // Now allowed. + EXPECT_EQ(++expected_nacks_sent, sent_nacks_.size()); + } + + // Giving up after 10 tries. + clock_->AdvanceTimeMilliseconds(3000); + Flush(); + EXPECT_EQ(expected_nacks_sent, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, ResendPacketMaxRetries) { + NackModule2& nack_module = CreateNackModule(TimeDelta::Millis(1)); + nack_module.OnReceivedPacket(1, false, false); + nack_module.OnReceivedPacket(3, false, false); + ASSERT_EQ(1u, sent_nacks_.size()); + EXPECT_EQ(2, sent_nacks_[0]); + + int backoff_factor = 1; + for (size_t retries = 1; retries < 10; ++retries) { + // Exponential backoff, so that we don't reject NACK because of time. + clock_->AdvanceTimeMilliseconds(backoff_factor * kDefaultRttMs); + backoff_factor *= 2; + WaitForSendNack(); + EXPECT_EQ(retries + 1, sent_nacks_.size()); + } + + clock_->AdvanceTimeMilliseconds(backoff_factor * kDefaultRttMs); + Flush(); + EXPECT_EQ(10u, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, TooLargeNackList) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(0, false, false); + nack_module.OnReceivedPacket(1001, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(0, keyframes_requested_); + nack_module.OnReceivedPacket(1003, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(1, keyframes_requested_); + nack_module.OnReceivedPacket(1004, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(1, keyframes_requested_); +} + +TEST_P(TestNackModule2, TooLargeNackListWithKeyFrame) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(0, false, false); + nack_module.OnReceivedPacket(1, true, false); + nack_module.OnReceivedPacket(1001, false, false); + EXPECT_EQ(999u, sent_nacks_.size()); + EXPECT_EQ(0, keyframes_requested_); + nack_module.OnReceivedPacket(1003, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(0, keyframes_requested_); + nack_module.OnReceivedPacket(1005, false, false); + EXPECT_EQ(1000u, sent_nacks_.size()); + EXPECT_EQ(1, keyframes_requested_); +} + +TEST_P(TestNackModule2, ClearUpTo) { + NackModule2& nack_module = CreateNackModule(TimeDelta::Millis(1)); + nack_module.OnReceivedPacket(0, false, false); + nack_module.OnReceivedPacket(100, false, false); + EXPECT_EQ(99u, sent_nacks_.size()); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module.ClearUpTo(50); + WaitForSendNack(); + ASSERT_EQ(50u, sent_nacks_.size()); + EXPECT_EQ(50, sent_nacks_[0]); +} + +TEST_P(TestNackModule2, ClearUpToWrap) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(0xfff0, false, false); + nack_module.OnReceivedPacket(0xf, false, false); + EXPECT_EQ(30u, sent_nacks_.size()); + + sent_nacks_.clear(); + clock_->AdvanceTimeMilliseconds(100); + nack_module.ClearUpTo(0); + WaitForSendNack(); + ASSERT_EQ(15u, sent_nacks_.size()); + EXPECT_EQ(0, sent_nacks_[0]); +} + +TEST_P(TestNackModule2, PacketNackCount) { + NackModule2& nack_module = CreateNackModule(TimeDelta::Millis(1)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(0, false, false)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(2, false, false)); + EXPECT_EQ(1, nack_module.OnReceivedPacket(1, false, false)); + + sent_nacks_.clear(); + nack_module.UpdateRtt(100); + EXPECT_EQ(0, nack_module.OnReceivedPacket(5, false, false)); + clock_->AdvanceTimeMilliseconds(100); + WaitForSendNack(); + EXPECT_EQ(4u, sent_nacks_.size()); + + clock_->AdvanceTimeMilliseconds(125); + WaitForSendNack(); + + EXPECT_EQ(6u, sent_nacks_.size()); + + EXPECT_EQ(3, nack_module.OnReceivedPacket(3, false, false)); + EXPECT_EQ(3, nack_module.OnReceivedPacket(4, false, false)); + EXPECT_EQ(0, nack_module.OnReceivedPacket(4, false, false)); +} + +TEST_P(TestNackModule2, NackListFullAndNoOverlapWithKeyframes) { + NackModule2& nack_module = CreateNackModule(); + const int kMaxNackPackets = 1000; + const unsigned int kFirstGap = kMaxNackPackets - 20; + const unsigned int kSecondGap = 200; + uint16_t seq_num = 0; + nack_module.OnReceivedPacket(seq_num++, true, false); + seq_num += kFirstGap; + nack_module.OnReceivedPacket(seq_num++, true, false); + EXPECT_EQ(kFirstGap, sent_nacks_.size()); + sent_nacks_.clear(); + seq_num += kSecondGap; + nack_module.OnReceivedPacket(seq_num, true, false); + EXPECT_EQ(kSecondGap, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, HandleFecRecoveredPacket) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(1, false, false); + nack_module.OnReceivedPacket(4, false, true); + EXPECT_EQ(0u, sent_nacks_.size()); + nack_module.OnReceivedPacket(5, false, false); + EXPECT_EQ(2u, sent_nacks_.size()); +} + +TEST_P(TestNackModule2, SendNackWithoutDelay) { + NackModule2& nack_module = CreateNackModule(); + nack_module.OnReceivedPacket(0, false, false); + nack_module.OnReceivedPacket(100, false, false); + EXPECT_EQ(99u, sent_nacks_.size()); +} + +INSTANTIATE_TEST_SUITE_P(WithAndWithoutBackoff, + TestNackModule2, + ::testing::Values(true, false)); + +class TestNackModule2WithFieldTrial : public ::testing::Test, + public NackSender, + public KeyFrameRequestSender { + protected: + TestNackModule2WithFieldTrial() + : nack_delay_field_trial_("WebRTC-SendNackDelayMs/10/"), + clock_(new SimulatedClock(0)), + nack_module_(TaskQueueBase::Current(), clock_.get(), this, this), + keyframes_requested_(0) {} + + void SendNack(const std::vector<uint16_t>& sequence_numbers, + bool buffering_allowed) override { + sent_nacks_.insert(sent_nacks_.end(), sequence_numbers.begin(), + sequence_numbers.end()); + } + + void RequestKeyFrame() override { ++keyframes_requested_; } + + test::ScopedFieldTrials nack_delay_field_trial_; + std::unique_ptr<SimulatedClock> clock_; + NackModule2 nack_module_; + std::vector<uint16_t> sent_nacks_; + int keyframes_requested_; +}; + +TEST_F(TestNackModule2WithFieldTrial, SendNackWithDelay) { + nack_module_.OnReceivedPacket(0, false, false); + nack_module_.OnReceivedPacket(100, false, false); + EXPECT_EQ(0u, sent_nacks_.size()); + clock_->AdvanceTimeMilliseconds(10); + nack_module_.OnReceivedPacket(106, false, false); + EXPECT_EQ(99u, sent_nacks_.size()); + clock_->AdvanceTimeMilliseconds(10); + nack_module_.OnReceivedPacket(109, false, false); + EXPECT_EQ(104u, sent_nacks_.size()); +} +} // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/nack_module_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/nack_module_unittest.cc index ab1c76f1b5c..f91eb750f02 100644 --- a/chromium/third_party/webrtc/modules/video_coding/nack_module_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/nack_module_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/video_coding/nack_module.h" +#include "modules/video_coding/deprecated/nack_module.h" #include <algorithm> #include <cstdint> @@ -45,7 +45,7 @@ class TestNackModule : public ::testing::TestWithParam<bool>, static constexpr int64_t kDefaultRttMs = 20; std::unique_ptr<SimulatedClock> clock_; test::ScopedFieldTrials field_trial_; - NackModule nack_module_; + DEPRECATED_NackModule nack_module_; std::vector<uint16_t> sent_nacks_; int keyframes_requested_; }; @@ -352,7 +352,7 @@ class TestNackModuleWithFieldTrial : public ::testing::Test, test::ScopedFieldTrials nack_delay_field_trial_; std::unique_ptr<SimulatedClock> clock_; - NackModule nack_module_; + DEPRECATED_NackModule nack_module_; std::vector<uint16_t> sent_nacks_; int keyframes_requested_; }; diff --git a/chromium/third_party/webrtc/modules/video_coding/packet_buffer.cc b/chromium/third_party/webrtc/modules/video_coding/packet_buffer.cc index 5db3c0f670a..7da8a1c3016 100644 --- a/chromium/third_party/webrtc/modules/video_coding/packet_buffer.cc +++ b/chromium/third_party/webrtc/modules/video_coding/packet_buffer.cc @@ -78,7 +78,7 @@ PacketBuffer::~PacketBuffer() { PacketBuffer::InsertResult PacketBuffer::InsertPacket( std::unique_ptr<PacketBuffer::Packet> packet) { PacketBuffer::InsertResult result; - rtc::CritScope lock(&crit_); + MutexLock lock(&mutex_); uint16_t seq_num = packet->seq_num; size_t index = seq_num % buffer_.size(); @@ -112,7 +112,7 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket( // Clear the buffer, delete payload, and return false to signal that a // new keyframe is needed. RTC_LOG(LS_WARNING) << "Clear PacketBuffer and request key frame."; - Clear(); + ClearInternal(); result.buffer_cleared = true; return result; } @@ -136,7 +136,7 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket( } void PacketBuffer::ClearTo(uint16_t seq_num) { - rtc::CritScope lock(&crit_); + MutexLock lock(&mutex_); // We have already cleared past this sequence number, no need to do anything. if (is_cleared_to_first_seq_num_ && AheadOf<uint16_t>(first_seq_num_, seq_num)) { @@ -173,37 +173,41 @@ void PacketBuffer::ClearTo(uint16_t seq_num) { } void PacketBuffer::Clear() { - rtc::CritScope lock(&crit_); - for (auto& entry : buffer_) { - entry = nullptr; - } - - first_packet_received_ = false; - is_cleared_to_first_seq_num_ = false; - last_received_packet_ms_.reset(); - last_received_keyframe_packet_ms_.reset(); - newest_inserted_seq_num_.reset(); - missing_packets_.clear(); + MutexLock lock(&mutex_); + ClearInternal(); } PacketBuffer::InsertResult PacketBuffer::InsertPadding(uint16_t seq_num) { PacketBuffer::InsertResult result; - rtc::CritScope lock(&crit_); + MutexLock lock(&mutex_); UpdateMissingPackets(seq_num); result.packets = FindFrames(static_cast<uint16_t>(seq_num + 1)); return result; } absl::optional<int64_t> PacketBuffer::LastReceivedPacketMs() const { - rtc::CritScope lock(&crit_); + MutexLock lock(&mutex_); return last_received_packet_ms_; } absl::optional<int64_t> PacketBuffer::LastReceivedKeyframePacketMs() const { - rtc::CritScope lock(&crit_); + MutexLock lock(&mutex_); return last_received_keyframe_packet_ms_; } +void PacketBuffer::ClearInternal() { + for (auto& entry : buffer_) { + entry = nullptr; + } + + first_packet_received_ = false; + is_cleared_to_first_seq_num_ = false; + last_received_packet_ms_.reset(); + last_received_keyframe_packet_ms_.reset(); + newest_inserted_seq_num_.reset(); + missing_packets_.clear(); +} + bool PacketBuffer::ExpandBufferSize() { if (buffer_.size() == max_size_) { RTC_LOG(LS_WARNING) << "PacketBuffer is already at max size (" << max_size_ @@ -359,15 +363,10 @@ std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames( VideoFrameType::kVideoFrameDelta; } - // With IPPP, if this is not a keyframe, make sure there are no gaps - // in the packet sequence numbers up until this point. - const uint8_t h264tid = - buffer_[start_index] != nullptr - ? buffer_[start_index]->video_header.frame_marking.temporal_id - : kNoTemporalIdx; - if (h264tid == kNoTemporalIdx && !is_h264_keyframe && - missing_packets_.upper_bound(start_seq_num) != - missing_packets_.begin()) { + // If this is not a keyframe, make sure there are no gaps in the packet + // sequence numbers up until this point. + if (!is_h264_keyframe && missing_packets_.upper_bound(start_seq_num) != + missing_packets_.begin()) { return found_frames; } } diff --git a/chromium/third_party/webrtc/modules/video_coding/packet_buffer.h b/chromium/third_party/webrtc/modules/video_coding/packet_buffer.h index c480e372395..508fa8395f3 100644 --- a/chromium/third_party/webrtc/modules/video_coding/packet_buffer.h +++ b/chromium/third_party/webrtc/modules/video_coding/packet_buffer.h @@ -22,8 +22,8 @@ #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "rtc_base/copy_on_write_buffer.h" -#include "rtc_base/critical_section.h" #include "rtc_base/numerics/sequence_number_util.h" +#include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" @@ -82,62 +82,68 @@ class PacketBuffer { PacketBuffer(Clock* clock, size_t start_buffer_size, size_t max_buffer_size); ~PacketBuffer(); - InsertResult InsertPacket(std::unique_ptr<Packet> packet) - ABSL_MUST_USE_RESULT; - InsertResult InsertPadding(uint16_t seq_num) ABSL_MUST_USE_RESULT; - void ClearTo(uint16_t seq_num); - void Clear(); + InsertResult InsertPacket(std::unique_ptr<Packet> packet) ABSL_MUST_USE_RESULT + RTC_LOCKS_EXCLUDED(mutex_); + InsertResult InsertPadding(uint16_t seq_num) ABSL_MUST_USE_RESULT + RTC_LOCKS_EXCLUDED(mutex_); + void ClearTo(uint16_t seq_num) RTC_LOCKS_EXCLUDED(mutex_); + void Clear() RTC_LOCKS_EXCLUDED(mutex_); // Timestamp (not RTP timestamp) of the last received packet/keyframe packet. - absl::optional<int64_t> LastReceivedPacketMs() const; - absl::optional<int64_t> LastReceivedKeyframePacketMs() const; + absl::optional<int64_t> LastReceivedPacketMs() const + RTC_LOCKS_EXCLUDED(mutex_); + absl::optional<int64_t> LastReceivedKeyframePacketMs() const + RTC_LOCKS_EXCLUDED(mutex_); private: Clock* const clock_; + // Clears with |mutex_| taken. + void ClearInternal() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Tries to expand the buffer. - bool ExpandBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + bool ExpandBufferSize() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Test if all previous packets has arrived for the given sequence number. bool PotentialNewFrame(uint16_t seq_num) const - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); // Test if all packets of a frame has arrived, and if so, returns packets to // create frames. std::vector<std::unique_ptr<Packet>> FindFrames(uint16_t seq_num) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); void UpdateMissingPackets(uint16_t seq_num) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - rtc::CriticalSection crit_; + mutable Mutex mutex_; // buffer_.size() and max_size_ must always be a power of two. const size_t max_size_; // The fist sequence number currently in the buffer. - uint16_t first_seq_num_ RTC_GUARDED_BY(crit_); + uint16_t first_seq_num_ RTC_GUARDED_BY(mutex_); // If the packet buffer has received its first packet. - bool first_packet_received_ RTC_GUARDED_BY(crit_); + bool first_packet_received_ RTC_GUARDED_BY(mutex_); // If the buffer is cleared to |first_seq_num_|. - bool is_cleared_to_first_seq_num_ RTC_GUARDED_BY(crit_); + bool is_cleared_to_first_seq_num_ RTC_GUARDED_BY(mutex_); // Buffer that holds the the inserted packets and information needed to // determine continuity between them. - std::vector<std::unique_ptr<Packet>> buffer_ RTC_GUARDED_BY(crit_); + std::vector<std::unique_ptr<Packet>> buffer_ RTC_GUARDED_BY(mutex_); // Timestamp of the last received packet/keyframe packet. - absl::optional<int64_t> last_received_packet_ms_ RTC_GUARDED_BY(crit_); + absl::optional<int64_t> last_received_packet_ms_ RTC_GUARDED_BY(mutex_); absl::optional<int64_t> last_received_keyframe_packet_ms_ - RTC_GUARDED_BY(crit_); + RTC_GUARDED_BY(mutex_); absl::optional<uint32_t> last_received_keyframe_rtp_timestamp_ - RTC_GUARDED_BY(crit_); + RTC_GUARDED_BY(mutex_); - absl::optional<uint16_t> newest_inserted_seq_num_ RTC_GUARDED_BY(crit_); + absl::optional<uint16_t> newest_inserted_seq_num_ RTC_GUARDED_BY(mutex_); std::set<uint16_t, DescendingSeqNumComp<uint16_t>> missing_packets_ - RTC_GUARDED_BY(crit_); + RTC_GUARDED_BY(mutex_); // Indicates if we should require SPS, PPS, and IDR for a particular // RTP timestamp to treat the corresponding frame as a keyframe. diff --git a/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder.cc b/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder.cc index bdef991b8b4..2a43c275d6d 100644 --- a/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder.cc +++ b/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder.cc @@ -108,8 +108,6 @@ RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) { return ManageFrameVp8(frame); case kVideoCodecVP9: return ManageFrameVp9(frame); - case kVideoCodecH264: - return ManageFrameH264(frame); case kVideoCodecGeneric: if (auto* generic_header = absl::get_if<RTPVideoHeaderLegacyGeneric>( &frame->GetRtpVideoHeader().video_type_header)) { @@ -715,130 +713,6 @@ void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) { frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id); } -RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameH264( - RtpFrameObject* frame) { - const FrameMarking& rtp_frame_marking = frame->GetFrameMarking(); - - uint8_t tid = rtp_frame_marking.temporal_id; - bool blSync = rtp_frame_marking.base_layer_sync; - - if (tid == kNoTemporalIdx) - return ManageFramePidOrSeqNum(std::move(frame), kNoPictureId); - - // Protect against corrupted packets with arbitrary large temporal idx. - if (tid >= kMaxTemporalLayers) - return kDrop; - - frame->id.picture_id = frame->last_seq_num(); - - if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { - // For H264, use last_seq_num_gop_ to simply store last picture id - // as a pair of unpadded and padded sequence numbers. - if (last_seq_num_gop_.empty()) { - last_seq_num_gop_.insert(std::make_pair( - 0, std::make_pair(frame->id.picture_id, frame->id.picture_id))); - } - } - - // Stash if we have no keyframe yet. - if (last_seq_num_gop_.empty()) - return kStash; - - // Check for gap in sequence numbers. Store in |not_yet_received_seq_num_|. - if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) { - uint16_t last_pic_id_padded = last_seq_num_gop_.begin()->second.second; - if (AheadOf<uint16_t>(frame->id.picture_id, last_pic_id_padded)) { - do { - last_pic_id_padded = last_pic_id_padded + 1; - not_yet_received_seq_num_.insert(last_pic_id_padded); - } while (last_pic_id_padded != frame->id.picture_id); - } - } - - int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(rtp_frame_marking.tl0_pic_idx); - - // Clean up info for base layers that are too old. - int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo; - auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx); - layer_info_.erase(layer_info_.begin(), clean_layer_info_to); - - // Clean up info about not yet received frames that are too old. - uint16_t old_picture_id = frame->id.picture_id - kMaxNotYetReceivedFrames * 2; - auto clean_frames_to = not_yet_received_seq_num_.lower_bound(old_picture_id); - not_yet_received_seq_num_.erase(not_yet_received_seq_num_.begin(), - clean_frames_to); - - if (frame->frame_type() == VideoFrameType::kVideoFrameKey) { - frame->num_references = 0; - layer_info_[unwrapped_tl0].fill(-1); - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; - } - - auto layer_info_it = - layer_info_.find(tid == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0); - - // Stash if we have no base layer frame yet. - if (layer_info_it == layer_info_.end()) - return kStash; - - // Base layer frame. Copy layer info from previous base layer frame. - if (tid == 0) { - layer_info_it = - layer_info_.insert(std::make_pair(unwrapped_tl0, layer_info_it->second)) - .first; - frame->num_references = 1; - frame->references[0] = layer_info_it->second[0]; - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; - } - - // This frame only references its base layer frame. - if (blSync) { - frame->num_references = 1; - frame->references[0] = layer_info_it->second[0]; - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; - } - - // Find all references for general frame. - frame->num_references = 0; - for (uint8_t layer = 0; layer <= tid; ++layer) { - // Stash if we have not yet received frames on this temporal layer. - if (layer_info_it->second[layer] == -1) - return kStash; - - // Drop if the last frame on this layer is ahead of this frame. A layer sync - // frame was received after this frame for the same base layer frame. - uint16_t last_frame_in_layer = layer_info_it->second[layer]; - if (AheadOf<uint16_t>(last_frame_in_layer, frame->id.picture_id)) - return kDrop; - - // Stash and wait for missing frame between this frame and the reference - auto not_received_seq_num_it = - not_yet_received_seq_num_.upper_bound(last_frame_in_layer); - if (not_received_seq_num_it != not_yet_received_seq_num_.end() && - AheadOf<uint16_t>(frame->id.picture_id, *not_received_seq_num_it)) { - return kStash; - } - - if (!(AheadOf<uint16_t>(frame->id.picture_id, last_frame_in_layer))) { - RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id - << " and packet range [" << frame->first_seq_num() - << ", " << frame->last_seq_num() - << "] already received, " - " dropping frame."; - return kDrop; - } - - ++frame->num_references; - frame->references[layer] = last_frame_in_layer; - } - - UpdateDataH264(frame, unwrapped_tl0, tid); - return kHandOff; -} - void RtpFrameReferenceFinder::UpdateLastPictureIdWithPaddingH264() { auto seq_num_it = last_seq_num_gop_.begin(); diff --git a/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc index 9ded6bcb964..0c08ddd3027 100644 --- a/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/rtp_frame_reference_finder_unittest.cc @@ -32,13 +32,11 @@ std::unique_ptr<RtpFrameObject> CreateFrame( uint16_t seq_num_end, bool keyframe, VideoCodecType codec, - const RTPVideoTypeHeader& video_type_header, - const FrameMarking& frame_markings) { + const RTPVideoTypeHeader& video_type_header) { RTPVideoHeader video_header; video_header.frame_type = keyframe ? VideoFrameType::kVideoFrameKey : VideoFrameType::kVideoFrameDelta; video_header.video_type_header = video_type_header; - video_header.frame_marking = frame_markings; // clang-format off return std::make_unique<RtpFrameObject>( @@ -92,7 +90,7 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, bool keyframe) { std::unique_ptr<RtpFrameObject> frame = CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecGeneric, - RTPVideoTypeHeader(), FrameMarking()); + RTPVideoTypeHeader()); reference_finder_->ManageFrame(std::move(frame)); } @@ -110,9 +108,8 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, vp8_header.tl0PicIdx = tl0; vp8_header.layerSync = sync; - std::unique_ptr<RtpFrameObject> frame = - CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecVP8, - vp8_header, FrameMarking()); + std::unique_ptr<RtpFrameObject> frame = CreateFrame( + seq_num_start, seq_num_end, keyframe, kVideoCodecVP8, vp8_header); reference_finder_->ManageFrame(std::move(frame)); } @@ -140,9 +137,8 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, vp9_header.gof = *ss; } - std::unique_ptr<RtpFrameObject> frame = - CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, - vp9_header, FrameMarking()); + std::unique_ptr<RtpFrameObject> frame = CreateFrame( + seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, vp9_header); reference_finder_->ManageFrame(std::move(frame)); } @@ -166,26 +162,15 @@ class TestRtpFrameReferenceFinder : public ::testing::Test, for (size_t i = 0; i < refs.size(); ++i) vp9_header.pid_diff[i] = refs[i]; - std::unique_ptr<RtpFrameObject> frame = - CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, - vp9_header, FrameMarking()); + std::unique_ptr<RtpFrameObject> frame = CreateFrame( + seq_num_start, seq_num_end, keyframe, kVideoCodecVP9, vp9_header); reference_finder_->ManageFrame(std::move(frame)); } - void InsertH264(uint16_t seq_num_start, - uint16_t seq_num_end, - bool keyframe, - uint8_t tid = kNoTemporalIdx, - int32_t tl0 = kNoTl0PicIdx, - bool sync = false) { - FrameMarking frame_marking{}; - frame_marking.temporal_id = tid; - frame_marking.tl0_pic_idx = tl0; - frame_marking.base_layer_sync = sync; - + void InsertH264(uint16_t seq_num_start, uint16_t seq_num_end, bool keyframe) { std::unique_ptr<RtpFrameObject> frame = CreateFrame(seq_num_start, seq_num_end, keyframe, kVideoCodecH264, - RTPVideoTypeHeader(), frame_marking); + RTPVideoTypeHeader()); reference_finder_->ManageFrame(std::move(frame)); } @@ -1440,53 +1425,46 @@ TEST_F(TestRtpFrameReferenceFinder, H264KeyFrameReferences) { CheckReferencesH264(sn); } -// Test with 1 temporal layer. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_0) { - uint16_t sn = Rand(); +TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrap) { + uint16_t sn = 0xFFFF; - InsertH264(sn, sn, true, 0, 1); - InsertH264(sn + 1, sn + 1, false, 0, 2); - InsertH264(sn + 2, sn + 2, false, 0, 3); - InsertH264(sn + 3, sn + 3, false, 0, 4); + InsertH264(sn - 1, sn - 1, true); + InsertH264(sn, sn, false); + InsertH264(sn + 1, sn + 1, false); + InsertH264(sn + 2, sn + 2, false); ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn); + CheckReferencesH264(sn - 1); + CheckReferencesH264(sn, sn - 1); CheckReferencesH264(sn + 1, sn); CheckReferencesH264(sn + 2, sn + 1); - CheckReferencesH264(sn + 3, sn + 2); } -TEST_F(TestRtpFrameReferenceFinder, H264DuplicateTl1Frames) { +TEST_F(TestRtpFrameReferenceFinder, H264Frames) { uint16_t sn = Rand(); - InsertH264(sn, sn, true, 0, 0); - InsertH264(sn + 1, sn + 1, false, 1, 0, true); - InsertH264(sn + 2, sn + 2, false, 0, 1); - InsertH264(sn + 3, sn + 3, false, 1, 1); - InsertH264(sn + 3, sn + 3, false, 1, 1); - InsertH264(sn + 4, sn + 4, false, 0, 2); - InsertH264(sn + 5, sn + 5, false, 1, 2); + InsertH264(sn, sn, true); + InsertH264(sn + 1, sn + 1, false); + InsertH264(sn + 2, sn + 2, false); + InsertH264(sn + 3, sn + 3, false); - ASSERT_EQ(6UL, frames_from_callback_.size()); + ASSERT_EQ(4UL, frames_from_callback_.size()); CheckReferencesH264(sn); CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn + 2); - CheckReferencesH264(sn + 5, sn + 3, sn + 4); + CheckReferencesH264(sn + 2, sn + 1); + CheckReferencesH264(sn + 3, sn + 2); } -// Test with 1 temporal layer. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_0) { +TEST_F(TestRtpFrameReferenceFinder, H264Reordering) { uint16_t sn = Rand(); - InsertH264(sn, sn, true, 0, 1); - InsertH264(sn + 1, sn + 1, false, 0, 2); - InsertH264(sn + 3, sn + 3, false, 0, 4); - InsertH264(sn + 2, sn + 2, false, 0, 3); - InsertH264(sn + 5, sn + 5, false, 0, 6); - InsertH264(sn + 6, sn + 6, false, 0, 7); - InsertH264(sn + 4, sn + 4, false, 0, 5); + InsertH264(sn, sn, true); + InsertH264(sn + 1, sn + 1, false); + InsertH264(sn + 3, sn + 3, false); + InsertH264(sn + 2, sn + 2, false); + InsertH264(sn + 5, sn + 5, false); + InsertH264(sn + 6, sn + 6, false); + InsertH264(sn + 4, sn + 4, false); ASSERT_EQ(7UL, frames_from_callback_.size()); CheckReferencesH264(sn); @@ -1498,258 +1476,13 @@ TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_0) { CheckReferencesH264(sn + 6, sn + 5); } -// Test with 2 temporal layers in a 01 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_01) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 255); - InsertH264(sn + 1, sn + 1, false, 1, 255, true); - InsertH264(sn + 2, sn + 2, false, 0, 0); - InsertH264(sn + 3, sn + 3, false, 1, 0); - - ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn + 1, sn + 2); -} - -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersMultiSn_01) { - uint16_t sn = Rand(); - - InsertH264(sn, sn + 3, true, 0, 255); - InsertH264(sn + 4, sn + 5, false, 1, 255, true); - InsertH264(sn + 6, sn + 8, false, 0, 0); - InsertH264(sn + 9, sn + 9, false, 1, 0); - - ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn + 3); - CheckReferencesH264(sn + 5, sn + 3); - CheckReferencesH264(sn + 8, sn + 3); - CheckReferencesH264(sn + 9, sn + 5, sn + 8); -} - -// Test with 2 temporal layers in a 01 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_01) { - uint16_t sn = Rand(); - - InsertH264(sn + 1, sn + 1, false, 1, 255, true); - InsertH264(sn, sn, true, 0, 255); - InsertH264(sn + 3, sn + 3, false, 1, 0); - InsertH264(sn + 5, sn + 5, false, 1, 1); - InsertH264(sn + 2, sn + 2, false, 0, 0); - InsertH264(sn + 4, sn + 4, false, 0, 1); - InsertH264(sn + 6, sn + 6, false, 0, 2); - InsertH264(sn + 7, sn + 7, false, 1, 2); - - ASSERT_EQ(8UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn + 2); - CheckReferencesH264(sn + 5, sn + 3, sn + 4); - CheckReferencesH264(sn + 6, sn + 4); - CheckReferencesH264(sn + 7, sn + 5, sn + 6); -} - -// Test with 3 temporal layers in a 0212 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayers_0212) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 55); - InsertH264(sn + 1, sn + 1, false, 2, 55, true); - InsertH264(sn + 2, sn + 2, false, 1, 55, true); - InsertH264(sn + 3, sn + 3, false, 2, 55); - InsertH264(sn + 4, sn + 4, false, 0, 56); - InsertH264(sn + 5, sn + 5, false, 2, 56, true); - InsertH264(sn + 6, sn + 6, false, 1, 56, true); - InsertH264(sn + 7, sn + 7, false, 2, 56); - InsertH264(sn + 8, sn + 8, false, 0, 57); - InsertH264(sn + 9, sn + 9, false, 2, 57, true); - InsertH264(sn + 10, sn + 10, false, 1, 57, true); - InsertH264(sn + 11, sn + 11, false, 2, 57); - - ASSERT_EQ(12UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn); - CheckReferencesH264(sn + 5, sn + 4); - CheckReferencesH264(sn + 6, sn + 4); - CheckReferencesH264(sn + 7, sn + 4, sn + 5, sn + 6); - CheckReferencesH264(sn + 8, sn + 4); - CheckReferencesH264(sn + 9, sn + 8); - CheckReferencesH264(sn + 10, sn + 8); - CheckReferencesH264(sn + 11, sn + 8, sn + 9, sn + 10); -} - -// Test with 3 temporal layers in a 0212 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersMissingFrame_0212) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 55, false); - InsertH264(sn + 2, sn + 2, false, 1, 55, true); - InsertH264(sn + 3, sn + 3, false, 2, 55, false); - - ASSERT_EQ(2UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 2, sn); -} - -// Test with 3 temporal layers in a 0212 pattern. -TEST_F(TestRtpFrameReferenceFinder, H264TemporalLayersReordering_0212) { - uint16_t sn = Rand(); - - InsertH264(sn + 1, sn + 1, false, 2, 55, true); - InsertH264(sn, sn, true, 0, 55, false); - InsertH264(sn + 2, sn + 2, false, 1, 55, true); - InsertH264(sn + 4, sn + 4, false, 0, 56, false); - InsertH264(sn + 5, sn + 5, false, 2, 56, false); - InsertH264(sn + 3, sn + 3, false, 2, 55, false); - InsertH264(sn + 7, sn + 7, false, 2, 56, false); - InsertH264(sn + 9, sn + 9, false, 2, 57, true); - InsertH264(sn + 6, sn + 6, false, 1, 56, false); - InsertH264(sn + 8, sn + 8, false, 0, 57, false); - InsertH264(sn + 11, sn + 11, false, 2, 57, false); - InsertH264(sn + 10, sn + 10, false, 1, 57, true); - - ASSERT_EQ(12UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn, sn + 1, sn + 2); - CheckReferencesH264(sn + 4, sn); - CheckReferencesH264(sn + 5, sn + 2, sn + 3, sn + 4); - CheckReferencesH264(sn + 6, sn + 2, sn + 4); - CheckReferencesH264(sn + 7, sn + 4, sn + 5, sn + 6); - CheckReferencesH264(sn + 8, sn + 4); - CheckReferencesH264(sn + 9, sn + 8); - CheckReferencesH264(sn + 10, sn + 8); - CheckReferencesH264(sn + 11, sn + 8, sn + 9, sn + 10); -} - -TEST_F(TestRtpFrameReferenceFinder, H264InsertManyFrames_0212) { - uint16_t sn = Rand(); - - const int keyframes_to_insert = 50; - const int frames_per_keyframe = 120; // Should be a multiple of 4. - uint8_t tl0 = 128; - - for (int k = 0; k < keyframes_to_insert; ++k) { - InsertH264(sn, sn, true, 0, tl0, false); - InsertH264(sn + 1, sn + 1, false, 2, tl0, true); - InsertH264(sn + 2, sn + 2, false, 1, tl0, true); - InsertH264(sn + 3, sn + 3, false, 2, tl0, false); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 3, sn, sn + 1, sn + 2); - frames_from_callback_.clear(); - ++tl0; - - for (int f = 4; f < frames_per_keyframe; f += 4) { - uint16_t sf = sn + f; - - InsertH264(sf, sf, false, 0, tl0, false); - InsertH264(sf + 1, sf + 1, false, 2, tl0, false); - InsertH264(sf + 2, sf + 2, false, 1, tl0, false); - InsertH264(sf + 3, sf + 3, false, 2, tl0, false); - CheckReferencesH264(sf, sf - 4); - CheckReferencesH264(sf + 1, sf, sf - 1, sf - 2); - CheckReferencesH264(sf + 2, sf, sf - 2); - CheckReferencesH264(sf + 3, sf, sf + 1, sf + 2); - frames_from_callback_.clear(); - ++tl0; - } - - sn += frames_per_keyframe; - } -} - -TEST_F(TestRtpFrameReferenceFinder, H264LayerSync) { - uint16_t sn = Rand(); - - InsertH264(sn, sn, true, 0, 0, false); - InsertH264(sn + 1, sn + 1, false, 1, 0, true); - InsertH264(sn + 2, sn + 2, false, 0, 1, false); - ASSERT_EQ(3UL, frames_from_callback_.size()); - - InsertH264(sn + 4, sn + 4, false, 0, 2, false); - InsertH264(sn + 5, sn + 5, false, 1, 2, true); - InsertH264(sn + 6, sn + 6, false, 0, 3, false); - InsertH264(sn + 7, sn + 7, false, 1, 3, false); - - ASSERT_EQ(7UL, frames_from_callback_.size()); - CheckReferencesH264(sn); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn); - CheckReferencesH264(sn + 4, sn + 2); - CheckReferencesH264(sn + 5, sn + 4); - CheckReferencesH264(sn + 6, sn + 4); - CheckReferencesH264(sn + 7, sn + 6, sn + 5); -} - -TEST_F(TestRtpFrameReferenceFinder, H264Tl1SyncFrameAfterTl1Frame) { - InsertH264(1000, 1000, true, 0, 247, true); - InsertH264(1001, 1001, false, 0, 248, false); - InsertH264(1002, 1002, false, 1, 248, false); // Will be dropped - InsertH264(1003, 1003, false, 1, 248, true); // due to this frame. - - ASSERT_EQ(3UL, frames_from_callback_.size()); - CheckReferencesH264(1000); - CheckReferencesH264(1001, 1000); - CheckReferencesH264(1003, 1001); -} - -TEST_F(TestRtpFrameReferenceFinder, H264DetectMissingFrame_0212) { - InsertH264(1, 1, true, 0, 1, false); - InsertH264(2, 2, false, 2, 1, true); - InsertH264(3, 3, false, 1, 1, true); - InsertH264(4, 4, false, 2, 1, false); - - InsertH264(6, 6, false, 2, 2, false); - InsertH264(7, 7, false, 1, 2, false); - InsertH264(8, 8, false, 2, 2, false); - ASSERT_EQ(4UL, frames_from_callback_.size()); - - InsertH264(5, 5, false, 0, 2, false); - ASSERT_EQ(8UL, frames_from_callback_.size()); - - CheckReferencesH264(1); - CheckReferencesH264(2, 1); - CheckReferencesH264(3, 1); - CheckReferencesH264(4, 3, 2, 1); - - CheckReferencesH264(5, 1); - CheckReferencesH264(6, 5, 4, 3); - CheckReferencesH264(7, 5, 3); - CheckReferencesH264(8, 7, 6, 5); -} - -TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrap) { - uint16_t sn = 0xFFFF; - - InsertH264(sn - 1, sn - 1, true, 0, 1); - InsertH264(sn, sn, false, 0, 2); - InsertH264(sn + 1, sn + 1, false, 0, 3); - InsertH264(sn + 2, sn + 2, false, 0, 4); - - ASSERT_EQ(4UL, frames_from_callback_.size()); - CheckReferencesH264(sn - 1); - CheckReferencesH264(sn, sn - 1); - CheckReferencesH264(sn + 1, sn); - CheckReferencesH264(sn + 2, sn + 1); -} - TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrapMulti) { uint16_t sn = 0xFFFF; - InsertH264(sn - 3, sn - 2, true, 0, 1); - InsertH264(sn - 1, sn + 1, false, 0, 2); - InsertH264(sn + 2, sn + 3, false, 0, 3); - InsertH264(sn + 4, sn + 7, false, 0, 4); + InsertH264(sn - 3, sn - 2, true); + InsertH264(sn - 1, sn + 1, false); + InsertH264(sn + 2, sn + 3, false); + InsertH264(sn + 4, sn + 7, false); ASSERT_EQ(4UL, frames_from_callback_.size()); CheckReferencesH264(sn - 2); @@ -1758,35 +1491,5 @@ TEST_F(TestRtpFrameReferenceFinder, H264SequenceNumberWrapMulti) { CheckReferencesH264(sn + 7, sn + 3); } -TEST_F(TestRtpFrameReferenceFinder, H264Tl0PicIdxWrap) { - int numTl0Wraps = 1000; - int64_t sn = Rand(); - - for (int i = 0; i < numTl0Wraps; i++) { - for (int tl0 = 0; tl0 < 256; tl0 += 16, sn += 16) { - InsertH264(sn, sn, true, 0, tl0); - reference_finder_->ClearTo(sn); // Too many stashed frames cause errors. - - for (int k = 1; k < 8; k++) { - InsertH264(sn + k, sn + k, false, 0, tl0 + k); - } - - // Skip a TL0 index. - for (int k = 9; k < 16; k++) { - InsertH264(sn + k, sn + k, false, 0, tl0 + k); - } - - ASSERT_EQ(8UL, frames_from_callback_.size()); - - CheckReferencesH264(sn); - for (int k = 1; k < 8; k++) { - CheckReferencesH264(sn + k, sn + k - 1); - } - - frames_from_callback_.clear(); - } - } -} - } // namespace video_coding } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/session_info.cc b/chromium/third_party/webrtc/modules/video_coding/session_info.cc index e51d2936077..07b9a9d6b57 100644 --- a/chromium/third_party/webrtc/modules/video_coding/session_info.cc +++ b/chromium/third_party/webrtc/modules/video_coding/session_info.cc @@ -95,8 +95,6 @@ int VCMSessionInfo::TemporalId() const { return absl::get<RTPVideoHeaderVP9>( packets_.front().video_header.video_type_header) .temporal_idx; - } else if (packets_.front().video_header.codec == kVideoCodecH264) { - return packets_.front().video_header.frame_marking.temporal_id; } else { return kNoTemporalIdx; } @@ -113,8 +111,6 @@ bool VCMSessionInfo::LayerSync() const { return absl::get<RTPVideoHeaderVP9>( packets_.front().video_header.video_type_header) .temporal_up_switch; - } else if (packets_.front().video_header.codec == kVideoCodecH264) { - return packets_.front().video_header.frame_marking.base_layer_sync; } else { return false; } @@ -131,8 +127,6 @@ int VCMSessionInfo::Tl0PicId() const { return absl::get<RTPVideoHeaderVP9>( packets_.front().video_header.video_type_header) .tl0_pic_idx; - } else if (packets_.front().video_header.codec == kVideoCodecH264) { - return packets_.front().video_header.frame_marking.tl0_pic_idx; } else { return kNoTl0PicIdx; } diff --git a/chromium/third_party/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index db104c49d16..871e5a16924 100644 --- a/chromium/third_party/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -38,14 +38,16 @@ constexpr uint32_t kSimulcastScreenshareMaxBitrateKbps = 1250; class MockTemporalLayers : public Vp8FrameBufferController { public: - MOCK_METHOD2(NextFrameConfig, Vp8FrameConfig(size_t, uint32_t)); - MOCK_METHOD3(OnRatesUpdated, void(size_t, const std::vector<uint32_t>&, int)); - MOCK_METHOD1(UpdateConfiguration, Vp8EncoderConfig(size_t)); - MOCK_METHOD6(OnEncodeDone, - void(size_t, uint32_t, size_t, bool, int, CodecSpecificInfo*)); - MOCK_METHOD4(FrameEncoded, void(size_t, uint32_t, size_t, int)); - MOCK_CONST_METHOD0(Tl0PicIdx, uint8_t()); - MOCK_CONST_METHOD1(GetTemporalLayerId, int(const Vp8FrameConfig&)); + MOCK_METHOD(Vp8FrameConfig, NextFrameConfig, (size_t, uint32_t), (override)); + MOCK_METHOD(void, + OnRatesUpdated, + (size_t, const std::vector<uint32_t>&, int), + (override)); + MOCK_METHOD(Vp8EncoderConfig, UpdateConfiguration, (size_t), (override)); + MOCK_METHOD(void, + OnEncodeDone, + (size_t, uint32_t, size_t, bool, int, CodecSpecificInfo*), + (override)); }; } // namespace diff --git a/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc b/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc index 9c89235fe21..f8ddd4db41c 100644 --- a/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc +++ b/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.cc @@ -52,40 +52,65 @@ bool Vp9ReadSyncCode(rtc::BitBuffer* br) { return true; } -bool Vp9ReadColorConfig(rtc::BitBuffer* br, uint8_t profile) { - if (profile == 2 || profile == 3) { - // Bitdepth. - RETURN_FALSE_IF_ERROR(br->ConsumeBits(1)); +bool Vp9ReadColorConfig(rtc::BitBuffer* br, + uint8_t profile, + FrameInfo* frame_info) { + if (profile == 0 || profile == 1) { + frame_info->bit_detph = BitDept::k8Bit; + } else if (profile == 2 || profile == 3) { + uint32_t ten_or_twelve_bits; + RETURN_FALSE_IF_ERROR(br->ReadBits(&ten_or_twelve_bits, 1)); + frame_info->bit_detph = + ten_or_twelve_bits ? BitDept::k12Bit : BitDept::k10Bit; } uint32_t color_space; RETURN_FALSE_IF_ERROR(br->ReadBits(&color_space, 3)); + frame_info->color_space = static_cast<ColorSpace>(color_space); // SRGB is 7. if (color_space != 7) { - // YUV range flag. - RETURN_FALSE_IF_ERROR(br->ConsumeBits(1)); + uint32_t color_range; + RETURN_FALSE_IF_ERROR(br->ReadBits(&color_range, 1)); + frame_info->color_range = + color_range ? ColorRange::kFull : ColorRange::kStudio; + if (profile == 1 || profile == 3) { - // 1 bit: subsampling x. - // 1 bit: subsampling y. - RETURN_FALSE_IF_ERROR(br->ConsumeBits(2)); + uint32_t subsampling_x; + uint32_t subsampling_y; + RETURN_FALSE_IF_ERROR(br->ReadBits(&subsampling_x, 1)); + RETURN_FALSE_IF_ERROR(br->ReadBits(&subsampling_y, 1)); + if (subsampling_x) { + frame_info->sub_sampling = + subsampling_y ? YuvSubsampling::k420 : YuvSubsampling::k422; + } else { + frame_info->sub_sampling = + subsampling_y ? YuvSubsampling::k440 : YuvSubsampling::k444; + } + uint32_t reserved_bit; RETURN_FALSE_IF_ERROR(br->ReadBits(&reserved_bit, 1)); if (reserved_bit) { - RTC_LOG(LS_WARNING) << "Failed to get QP. Reserved bit set."; + RTC_LOG(LS_WARNING) << "Failed to parse header. Reserved bit set."; return false; } + } else { + // Profile 0 or 2. + frame_info->sub_sampling = YuvSubsampling::k420; } } else { + // SRGB + frame_info->color_range = ColorRange::kFull; if (profile == 1 || profile == 3) { + frame_info->sub_sampling = YuvSubsampling::k444; uint32_t reserved_bit; RETURN_FALSE_IF_ERROR(br->ReadBits(&reserved_bit, 1)); if (reserved_bit) { - RTC_LOG(LS_WARNING) << "Failed to get QP. Reserved bit set."; + RTC_LOG(LS_WARNING) << "Failed to parse header. Reserved bit set."; return false; } } else { - RTC_LOG(LS_WARNING) << "Failed to get QP. 4:4:4 color not supported in " - "profile 0 or 2."; + RTC_LOG(LS_WARNING) << "Failed to parse header. 4:4:4 color not supported" + " in profile 0 or 2."; return false; } } @@ -93,24 +118,38 @@ bool Vp9ReadColorConfig(rtc::BitBuffer* br, uint8_t profile) { return true; } -bool Vp9ReadFrameSize(rtc::BitBuffer* br) { - // 2 bytes: frame width. - // 2 bytes: frame height. - return br->ConsumeBytes(4); +bool Vp9ReadFrameSize(rtc::BitBuffer* br, FrameInfo* frame_info) { + // 16 bits: frame width - 1. + uint16_t frame_width_minus_one; + RETURN_FALSE_IF_ERROR(br->ReadUInt16(&frame_width_minus_one)); + // 16 bits: frame height - 1. + uint16_t frame_height_minus_one; + RETURN_FALSE_IF_ERROR(br->ReadUInt16(&frame_height_minus_one)); + frame_info->frame_width = frame_width_minus_one + 1; + frame_info->frame_height = frame_height_minus_one + 1; + return true; } -bool Vp9ReadRenderSize(rtc::BitBuffer* br) { - uint32_t bit; - RETURN_FALSE_IF_ERROR(br->ReadBits(&bit, 1)); - if (bit) { - // 2 bytes: render width. - // 2 bytes: render height. - RETURN_FALSE_IF_ERROR(br->ConsumeBytes(4)); +bool Vp9ReadRenderSize(rtc::BitBuffer* br, FrameInfo* frame_info) { + uint32_t render_and_frame_size_different; + RETURN_FALSE_IF_ERROR(br->ReadBits(&render_and_frame_size_different, 1)); + if (render_and_frame_size_different) { + // 16 bits: render width - 1. + uint16_t render_width_minus_one; + RETURN_FALSE_IF_ERROR(br->ReadUInt16(&render_width_minus_one)); + // 16 bits: render height - 1. + uint16_t render_height_minus_one; + RETURN_FALSE_IF_ERROR(br->ReadUInt16(&render_height_minus_one)); + frame_info->render_width = render_width_minus_one + 1; + frame_info->render_height = render_height_minus_one + 1; + } else { + frame_info->render_width = frame_info->frame_width; + frame_info->render_height = frame_info->frame_height; } return true; } -bool Vp9ReadFrameSizeFromRefs(rtc::BitBuffer* br) { +bool Vp9ReadFrameSizeFromRefs(rtc::BitBuffer* br, FrameInfo* frame_info) { uint32_t found_ref = 0; for (size_t i = 0; i < kVp9NumRefsPerFrame; i++) { // Size in refs. @@ -120,11 +159,11 @@ bool Vp9ReadFrameSizeFromRefs(rtc::BitBuffer* br) { } if (!found_ref) { - if (!Vp9ReadFrameSize(br)) { + if (!Vp9ReadFrameSize(br, frame_info)) { return false; } } - return Vp9ReadRenderSize(br); + return Vp9ReadRenderSize(br, frame_info); } bool Vp9ReadInterpolationFilter(rtc::BitBuffer* br) { @@ -166,14 +205,14 @@ bool Vp9ReadLoopfilter(rtc::BitBuffer* br) { } } // namespace -bool GetQp(const uint8_t* buf, size_t length, int* qp) { +bool Parse(const uint8_t* buf, size_t length, int* qp, FrameInfo* frame_info) { rtc::BitBuffer br(buf, length); // Frame marker. uint32_t frame_marker; RETURN_FALSE_IF_ERROR(br.ReadBits(&frame_marker, 2)); if (frame_marker != 0x2) { - RTC_LOG(LS_WARNING) << "Failed to get QP. Frame marker should be 2."; + RTC_LOG(LS_WARNING) << "Failed to parse header. Frame marker should be 2."; return false; } @@ -181,6 +220,7 @@ bool GetQp(const uint8_t* buf, size_t length, int* qp) { uint8_t profile; if (!Vp9ReadProfile(&br, &profile)) return false; + frame_info->profile = profile; // Show existing frame. uint32_t show_existing_frame; @@ -195,18 +235,21 @@ bool GetQp(const uint8_t* buf, size_t length, int* qp) { RETURN_FALSE_IF_ERROR(br.ReadBits(&frame_type, 1)); RETURN_FALSE_IF_ERROR(br.ReadBits(&show_frame, 1)); RETURN_FALSE_IF_ERROR(br.ReadBits(&error_resilient, 1)); + frame_info->show_frame = show_frame; + frame_info->error_resilient = error_resilient; - if (!frame_type) { + if (frame_type == 0) { + // Key-frame. if (!Vp9ReadSyncCode(&br)) return false; - if (!Vp9ReadColorConfig(&br, profile)) + if (!Vp9ReadColorConfig(&br, profile, frame_info)) return false; - if (!Vp9ReadFrameSize(&br)) + if (!Vp9ReadFrameSize(&br, frame_info)) return false; - if (!Vp9ReadRenderSize(&br)) + if (!Vp9ReadRenderSize(&br, frame_info)) return false; - } else { + // Non-keyframe. uint32_t intra_only = 0; if (!show_frame) RETURN_FALSE_IF_ERROR(br.ReadBits(&intra_only, 1)); @@ -218,14 +261,14 @@ bool GetQp(const uint8_t* buf, size_t length, int* qp) { return false; if (profile > 0) { - if (!Vp9ReadColorConfig(&br, profile)) + if (!Vp9ReadColorConfig(&br, profile, frame_info)) return false; } // Refresh frame flags. RETURN_FALSE_IF_ERROR(br.ConsumeBits(8)); - if (!Vp9ReadFrameSize(&br)) + if (!Vp9ReadFrameSize(&br, frame_info)) return false; - if (!Vp9ReadRenderSize(&br)) + if (!Vp9ReadRenderSize(&br, frame_info)) return false; } else { // Refresh frame flags. @@ -237,7 +280,7 @@ bool GetQp(const uint8_t* buf, size_t length, int* qp) { RETURN_FALSE_IF_ERROR(br.ConsumeBits(4)); } - if (!Vp9ReadFrameSizeFromRefs(&br)) + if (!Vp9ReadFrameSizeFromRefs(&br, frame_info)) return false; // Allow high precision mv. @@ -267,6 +310,20 @@ bool GetQp(const uint8_t* buf, size_t length, int* qp) { return true; } -} // namespace vp9 +bool GetQp(const uint8_t* buf, size_t length, int* qp) { + FrameInfo frame_info; + return Parse(buf, length, qp, &frame_info); +} +absl::optional<FrameInfo> ParseIntraFrameInfo(const uint8_t* buf, + size_t length) { + int qp = 0; + FrameInfo frame_info; + if (Parse(buf, length, &qp, &frame_info) && frame_info.frame_width > 0) { + return frame_info; + } + return absl::nullopt; +} + +} // namespace vp9 } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h b/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h index 69e8de87df0..a7f04670d22 100644 --- a/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h +++ b/chromium/third_party/webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h @@ -13,6 +13,7 @@ #include <stddef.h> #include <stdint.h> +#include "absl/types/optional.h" namespace webrtc { @@ -22,6 +23,65 @@ namespace vp9 { // Returns true on success, false otherwise. bool GetQp(const uint8_t* buf, size_t length, int* qp); +// Bit depth per channel. Support varies by profile. +enum class BitDept : uint8_t { + k8Bit = 8, + k10Bit = 10, + k12Bit = 12, +}; + +enum class ColorSpace : uint8_t { + CS_UNKNOWN = 0, // Unknown (in this case the color space must be signaled + // outside the VP9 bitstream). + CS_BT_601 = 1, // CS_BT_601 Rec. ITU-R BT.601-7 + CS_BT_709 = 2, // Rec. ITU-R BT.709-6 + CS_SMPTE_170 = 3, // SMPTE-170 + CS_SMPTE_240 = 4, // SMPTE-240 + CS_BT_2020 = 5, // Rec. ITU-R BT.2020-2 + CS_RESERVED = 6, // Reserved + CS_RGB = 7, // sRGB (IEC 61966-2-1) +}; + +enum class ColorRange { + kStudio, // Studio swing: + // For BitDepth equals 8: + // Y is between 16 and 235 inclusive. + // U and V are between 16 and 240 inclusive. + // For BitDepth equals 10: + // Y is between 64 and 940 inclusive. + // U and V are between 64 and 960 inclusive. + // For BitDepth equals 12: + // Y is between 256 and 3760. + // U and V are between 256 and 3840 inclusive. + kFull // Full swing; no restriction on Y, U, V values. +}; + +enum class YuvSubsampling { + k444, + k440, + k422, + k420, +}; + +struct FrameInfo { + int profile = 0; // Profile 0-3 are valid. + bool show_frame = false; + bool error_resilient = false; + BitDept bit_detph = BitDept::k8Bit; + ColorSpace color_space = ColorSpace::CS_UNKNOWN; + ColorRange color_range; + YuvSubsampling sub_sampling; + int frame_width = 0; + int frame_height = 0; + int render_width = 0; + int render_height = 0; +}; + +// Parses frame information for a VP9 key-frame or all-intra frame from a +// bitstream. Returns nullopt on failure or if not a key-frame. +absl::optional<FrameInfo> ParseIntraFrameInfo(const uint8_t* buf, + size_t length); + } // namespace vp9 } // namespace webrtc diff --git a/chromium/third_party/webrtc/modules/video_coding/video_receiver_unittest.cc b/chromium/third_party/webrtc/modules/video_coding/video_receiver_unittest.cc index 363838b846e..2872c8d2a9c 100644 --- a/chromium/third_party/webrtc/modules/video_coding/video_receiver_unittest.cc +++ b/chromium/third_party/webrtc/modules/video_coding/video_receiver_unittest.cc @@ -26,8 +26,10 @@ namespace { class MockPacketRequestCallback : public VCMPacketRequestCallback { public: - MOCK_METHOD2(ResendPackets, - int32_t(const uint16_t* sequenceNumbers, uint16_t length)); + MOCK_METHOD(int32_t, + ResendPackets, + (const uint16_t* sequenceNumbers, uint16_t length), + (override)); }; class MockVCMReceiveCallback : public VCMReceiveCallback { @@ -35,11 +37,12 @@ class MockVCMReceiveCallback : public VCMReceiveCallback { MockVCMReceiveCallback() {} virtual ~MockVCMReceiveCallback() {} - MOCK_METHOD4( - FrameToRender, - int32_t(VideoFrame&, absl::optional<uint8_t>, int32_t, VideoContentType)); - MOCK_METHOD1(OnIncomingPayloadType, void(int)); - MOCK_METHOD1(OnDecoderImplementationName, void(const char*)); + MOCK_METHOD(int32_t, + FrameToRender, + (VideoFrame&, absl::optional<uint8_t>, int32_t, VideoContentType), + (override)); + MOCK_METHOD(void, OnIncomingPayloadType, (int), (override)); + MOCK_METHOD(void, OnDecoderImplementationName, (const char*), (override)); }; class TestVideoReceiver : public ::testing::Test { |