diff options
Diffstat (limited to 'chromium/media/mojo')
51 files changed, 1389 insertions, 297 deletions
diff --git a/chromium/media/mojo/clients/BUILD.gn b/chromium/media/mojo/clients/BUILD.gn index 68fd43a30e6..6f470bae8ab 100644 --- a/chromium/media/mojo/clients/BUILD.gn +++ b/chromium/media/mojo/clients/BUILD.gn @@ -27,6 +27,8 @@ jumbo_source_set("clients") { # TODO(liberato): can we avoid this? "//content/test/*", + + "//third_party/blink/renderer/modules/webcodecs", ] sources = [ diff --git a/chromium/media/mojo/clients/mojo_cdm.cc b/chromium/media/mojo/clients/mojo_cdm.cc index acc9d1ca9de..df477ee2584 100644 --- a/chromium/media/mojo/clients/mojo_cdm.cc +++ b/chromium/media/mojo/clients/mojo_cdm.cc @@ -23,7 +23,6 @@ #include "media/mojo/mojom/decryptor.mojom.h" #include "services/service_manager/public/cpp/connect.h" #include "services/service_manager/public/mojom/interface_provider.mojom.h" -#include "url/origin.h" namespace media { @@ -39,7 +38,6 @@ void RecordConnectionError(bool connection_error_happened) { // static void MojoCdm::Create( const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, const SessionMessageCB& session_message_cb, @@ -55,8 +53,7 @@ void MojoCdm::Create( auto promise = std::make_unique<CdmInitializedPromise>( std::move(cdm_created_cb), mojo_cdm); - mojo_cdm->InitializeCdm(key_system, security_origin, cdm_config, - std::move(promise)); + mojo_cdm->InitializeCdm(key_system, cdm_config, std::move(promise)); } MojoCdm::MojoCdm(mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, @@ -103,7 +100,6 @@ MojoCdm::~MojoCdm() { // error handler can't be invoked and callbacks won't be dispatched. void MojoCdm::InitializeCdm(const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, std::unique_ptr<CdmInitializedPromise> promise) { DVLOG(1) << __func__ << ": " << key_system; @@ -127,7 +123,7 @@ void MojoCdm::InitializeCdm(const std::string& key_system, pending_init_promise_ = std::move(promise); remote_cdm_->Initialize( - key_system, security_origin, cdm_config, + key_system, cdm_config, base::BindOnce(&MojoCdm::OnCdmInitialized, base::Unretained(this))); } diff --git a/chromium/media/mojo/clients/mojo_cdm.h b/chromium/media/mojo/clients/mojo_cdm.h index 45f0fcb09f1..bee79ff2cf1 100644 --- a/chromium/media/mojo/clients/mojo_cdm.h +++ b/chromium/media/mojo/clients/mojo_cdm.h @@ -29,10 +29,6 @@ namespace base { class SingleThreadTaskRunner; } -namespace url { -class Origin; -} - namespace media { class MojoDecryptor; @@ -48,7 +44,6 @@ class MojoCdm : public ContentDecryptionModule, static void Create( const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, const SessionMessageCB& session_message_cb, @@ -94,7 +89,6 @@ class MojoCdm : public ContentDecryptionModule, ~MojoCdm() final; void InitializeCdm(const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, std::unique_ptr<CdmInitializedPromise> promise); diff --git a/chromium/media/mojo/clients/mojo_cdm_factory.cc b/chromium/media/mojo/clients/mojo_cdm_factory.cc index d606e735eb1..2d04068fb22 100644 --- a/chromium/media/mojo/clients/mojo_cdm_factory.cc +++ b/chromium/media/mojo/clients/mojo_cdm_factory.cc @@ -15,7 +15,6 @@ #include "media/mojo/clients/mojo_cdm.h" #include "media/mojo/mojom/interface_factory.mojom.h" #include "mojo/public/cpp/bindings/interface_request.h" -#include "url/origin.h" namespace media { @@ -29,7 +28,6 @@ MojoCdmFactory::~MojoCdmFactory() = default; void MojoCdmFactory::Create( const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, const SessionMessageCB& session_message_cb, const SessionClosedCB& session_closed_cb, @@ -38,13 +36,6 @@ void MojoCdmFactory::Create( CdmCreatedCB cdm_created_cb) { DVLOG(2) << __func__ << ": " << key_system; - if (security_origin.opaque()) { - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, - base::BindOnce(std::move(cdm_created_cb), nullptr, "Invalid origin.")); - return; - } - // If AesDecryptor can be used, always use it here in the local process. // Note: We should not run AesDecryptor in the browser process except for // testing. See http://crbug.com/441957. @@ -63,9 +54,8 @@ void MojoCdmFactory::Create( interface_factory_->CreateCdm( key_system, cdm_pending_remote.InitWithNewPipeAndPassReceiver()); - MojoCdm::Create(key_system, security_origin, cdm_config, - std::move(cdm_pending_remote), session_message_cb, - session_closed_cb, session_keys_change_cb, + MojoCdm::Create(key_system, cdm_config, std::move(cdm_pending_remote), + session_message_cb, session_closed_cb, session_keys_change_cb, session_expiration_update_cb, std::move(cdm_created_cb)); } diff --git a/chromium/media/mojo/clients/mojo_cdm_factory.h b/chromium/media/mojo/clients/mojo_cdm_factory.h index c3b8d055428..0d987bc51dd 100644 --- a/chromium/media/mojo/clients/mojo_cdm_factory.h +++ b/chromium/media/mojo/clients/mojo_cdm_factory.h @@ -21,7 +21,6 @@ class MojoCdmFactory : public CdmFactory { // CdmFactory implementation. void Create(const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, const SessionMessageCB& session_message_cb, const SessionClosedCB& session_closed_cb, diff --git a/chromium/media/mojo/clients/mojo_cdm_unittest.cc b/chromium/media/mojo/clients/mojo_cdm_unittest.cc index f7b402b4f54..99fd9646162 100644 --- a/chromium/media/mojo/clients/mojo_cdm_unittest.cc +++ b/chromium/media/mojo/clients/mojo_cdm_unittest.cc @@ -24,7 +24,6 @@ #include "mojo/public/cpp/bindings/receiver.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" -#include "url/origin.h" using ::testing::_; using ::testing::DoAll; @@ -48,7 +47,6 @@ namespace media { namespace { const char kClearKeyKeySystem[] = "org.w3.clearkey"; -const char kTestSecurityOrigin[] = "https://www.test.com"; // Random key ID used to create a session. const uint8_t kKeyId[] = { @@ -97,8 +95,8 @@ class MojoCdmTest : public ::testing::Test { } } - MojoCdm::Create(key_system, url::Origin::Create(GURL(kTestSecurityOrigin)), - CdmConfig(), cdm_receiver_.BindNewPipeAndPassRemote(), + MojoCdm::Create(key_system, CdmConfig(), + cdm_receiver_.BindNewPipeAndPassRemote(), base::Bind(&MockCdmClient::OnSessionMessage, base::Unretained(&cdm_client_)), base::Bind(&MockCdmClient::OnSessionClosed, @@ -125,8 +123,6 @@ class MojoCdmTest : public ::testing::Test { mojo_cdm_ = cdm; remote_cdm_ = cdm_factory_.GetCreatedCdm(); EXPECT_EQ(kClearKeyKeySystem, remote_cdm_->GetKeySystem()); - EXPECT_EQ(kTestSecurityOrigin, - remote_cdm_->GetSecurityOrigin().Serialize()); } void ForceConnectionError() { diff --git a/chromium/media/mojo/clients/mojo_renderer.cc b/chromium/media/mojo/clients/mojo_renderer.cc index b6d1a3d7777..aa60a7d2a76 100644 --- a/chromium/media/mojo/clients/mojo_renderer.cc +++ b/chromium/media/mojo/clients/mojo_renderer.cc @@ -10,6 +10,7 @@ #include "base/callback_helpers.h" #include "base/location.h" #include "base/single_thread_task_runner.h" +#include "media/base/cdm_context.h" #include "media/base/media_resource.h" #include "media/base/pipeline_status.h" #include "media/base/renderer_client.h" diff --git a/chromium/media/mojo/clients/mojo_renderer_unittest.cc b/chromium/media/mojo/clients/mojo_renderer_unittest.cc index 5925bc6aa88..f59e1be83bd 100644 --- a/chromium/media/mojo/clients/mojo_renderer_unittest.cc +++ b/chromium/media/mojo/clients/mojo_renderer_unittest.cc @@ -33,7 +33,6 @@ #include "mojo/public/cpp/bindings/self_owned_receiver.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" -#include "url/origin.h" using ::base::test::RunCallback; using ::base::test::RunOnceCallback; @@ -184,7 +183,6 @@ class MojoRendererTest : public ::testing::Test { void CreateCdm() { cdm_receiver_.Bind(cdm_remote_.BindNewPipeAndPassReceiver()); cdm_remote_->Initialize(kClearKeyKeySystem, - url::Origin::Create(GURL("https://www.test.com")), CdmConfig(), base::BindOnce(&MojoRendererTest::OnCdmCreated, base::Unretained(this))); diff --git a/chromium/media/mojo/clients/mojo_renderer_wrapper.cc b/chromium/media/mojo/clients/mojo_renderer_wrapper.cc index f23fade4203..2971244b9fa 100644 --- a/chromium/media/mojo/clients/mojo_renderer_wrapper.cc +++ b/chromium/media/mojo/clients/mojo_renderer_wrapper.cc @@ -14,9 +14,9 @@ MojoRendererWrapper::MojoRendererWrapper( MojoRendererWrapper::~MojoRendererWrapper() = default; -void MojoRendererWrapper::Initialize(media::MediaResource* media_resource, - media::RendererClient* client, - media::PipelineStatusCallback init_cb) { +void MojoRendererWrapper::Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) { mojo_renderer_->Initialize(media_resource, client, std::move(init_cb)); } @@ -36,8 +36,8 @@ void MojoRendererWrapper::SetVolume(float volume) { mojo_renderer_->SetVolume(volume); } -void MojoRendererWrapper::SetCdm(media::CdmContext* cdm_context, - media::CdmAttachedCB cdm_attached_cb) { +void MojoRendererWrapper::SetCdm(CdmContext* cdm_context, + CdmAttachedCB cdm_attached_cb) { mojo_renderer_->SetCdm(cdm_context, std::move(cdm_attached_cb)); } diff --git a/chromium/media/mojo/clients/mojo_renderer_wrapper.h b/chromium/media/mojo/clients/mojo_renderer_wrapper.h index 552a8f0513f..891b4ff0de7 100644 --- a/chromium/media/mojo/clients/mojo_renderer_wrapper.h +++ b/chromium/media/mojo/clients/mojo_renderer_wrapper.h @@ -13,8 +13,8 @@ namespace media { // Simple wrapper around a MojoRenderer. -// Provides a default behavior for forwarding all media::Renderer calls to a -// media::Renderer instance in a different process, through |mojo_renderer_|. +// Provides a default behavior for forwarding all Renderer calls to a +// Renderer instance in a different process, through |mojo_renderer_|. // Used as a base class to reduce boiler plate code for derived types, which can // override only the methods they need to specialize. class MojoRendererWrapper : public Renderer { @@ -24,7 +24,7 @@ class MojoRendererWrapper : public Renderer { // Renderer implementation. void Initialize(MediaResource* media_resource, - media::RendererClient* client, + RendererClient* client, PipelineStatusCallback init_cb) override; void SetCdm(CdmContext* cdm_context, CdmAttachedCB cdm_attached_cb) override; void SetLatencyHint(base::Optional<base::TimeDelta> latency_hint) override; diff --git a/chromium/media/mojo/mojom/BUILD.gn b/chromium/media/mojo/mojom/BUILD.gn index f86298e18d2..c7cb64d1dca 100644 --- a/chromium/media/mojo/mojom/BUILD.gn +++ b/chromium/media/mojo/mojom/BUILD.gn @@ -62,6 +62,7 @@ mojom("mojom") { "//gpu/ipc/common:interfaces", "//media/learning/mojo/public/mojom", "//mojo/public/mojom/base", + "//services/network/public/mojom", "//services/service_manager/public/mojom", "//ui/gfx/geometry/mojom", "//ui/gfx/mojom", @@ -79,11 +80,212 @@ mojom("mojom") { enabled_features = [ "enable_cast_renderer" ] } + shared_typemaps = [ + { + types = [ + { + mojom = "media.mojom.VideoFrameMetadata" + cpp = "::media::VideoFrameMetadata" + }, + ] + traits_headers = [ "video_frame_metadata_mojom_traits.h" ] + traits_public_deps = [ ":shared_mojom_traits" ] + }, + ] + + cpp_typemaps = [ + { + types = [ + { + mojom = "media.mojom.VideoRotation" + cpp = "::media::VideoRotation" + }, + ] + traits_headers = [ "media_types_enum_mojom_traits.h" ] + }, + { + types = [ + { + mojom = "media.mojom.PipelineStatistics" + cpp = "::media::PipelineStatistics" + }, + { + mojom = "media.mojom.PipelineDecoderInfo" + cpp = "::media::PipelineDecoderInfo" + }, + ] + traits_headers = [ "pipeline_status_mojom_traits.h" ] + }, + { + types = [ + { + mojom = "media.mojom.Status" + cpp = "::media::Status" + }, + ] + traits_headers = [ "status_mojom_traits.h" ] + traits_sources = [ "status_mojom_traits.cc" ] + }, + { + types = [ + { + mojom = "media.mojom.VideoColorSpace.PrimaryID" + cpp = "::media::VideoColorSpace::PrimaryID" + }, + { + mojom = "media.mojom.VideoColorSpace.TransferID:" + cpp = "::media::VideoColorSpace::TransferID" + }, + { + mojom = "media.mojom.VideoColorSpace.MatrixID:" + cpp = "::media::VideoColorSpace::MatrixID" + }, + { + mojom = "media.mojom.VideoColorSpace.RangeID:" + cpp = "::gfx::ColorSpace::RangeID" + }, + { + mojom = "media.mojom.VideoColorSpace:" + cpp = "::media::VideoColorSpace" + }, + ] + traits_headers = [ "video_color_space_mojom_traits.h" ] + }, + { + types = [ + { + mojom = "media.mojom.AudioCodec" + cpp = "::media::AudioCodec" + }, + { + mojom = "media.mojom.AudioCodecProfile" + cpp = "::media::AudioCodecProfile" + }, + { + mojom = "media.mojom.BufferingState" + cpp = "::media::BufferingState" + }, + { + mojom = "media.mojom.BufferingStateChangeReason" + cpp = "::media::BufferingStateChangeReason" + }, + { + mojom = "media.mojom.ChannelLayout" + cpp = "::media::ChannelLayout" + }, + { + mojom = "media.mojom.DecodeStatus" + cpp = "::media::DecodeStatus" + }, + { + mojom = "media.mojom.EncryptionScheme" + cpp = "::media::EncryptionScheme" + }, + { + mojom = "media.mojom.MediaContainerName" + cpp = "::media::container_names::MediaContainerName" + }, + { + mojom = "media.mojom.MediaLogRecord" + cpp = "::media::MediaLogRecord" + }, + { + mojom = "media.mojom.OutputDeviceStatus" + cpp = "::media::OutputDeviceStatus" + }, + { + mojom = "media.mojom.PipelineStatus" + cpp = "::media::PipelineStatus" + }, + { + mojom = "media.mojom.SampleFormat" + cpp = "::media::SampleFormat" + }, + { + mojom = "media.mojom.SubsampleEntry" + cpp = "::media::SubsampleEntry" + }, + { + mojom = "media.mojom.VideoCodec" + cpp = "::media::VideoCodec" + }, + { + mojom = "media.mojom.VideoCodecProfile" + cpp = "::media::VideoCodecProfile" + }, + { + mojom = "media.mojom.VideoPixelFormat" + cpp = "::media::VideoPixelFormat" + }, + { + mojom = "media.mojom.VideoTransformation" + cpp = "::media::VideoTransformation" + }, + { + mojom = "media.mojom.WaitingReason" + cpp = "::media::WaitingReason" + }, + { + mojom = "media.mojom.WatchTimeKey" + cpp = "::media::WatchTimeKey" + }, + { + mojom = "media.mojom.MediaStatusState" + cpp = "::media::MediaStatus::State" + }, + { + mojom = "media.mojom.StatusCode" + cpp = "::media::StatusCode" + }, + ] + traits_headers = [ + "video_transformation_mojom_traits.h", + "//media/base/ipc/media_param_traits_macros.h", + ] + traits_sources = [ "video_transformation_mojom_traits.cc" ] + }, + { + types = [ + { + mojom = "media.mojom.VideoFrame" + cpp = "::scoped_refptr<::media::VideoFrame>" + nullable_is_same_type = true + }, + ] + traits_headers = [ "video_frame_mojom_traits.h" ] + traits_sources = [ "video_frame_mojom_traits.cc" ] + traits_public_deps = [ + "//media/mojo/common:mojo_shared_buffer_video_frame", + "//ui/gfx/geometry/mojom", + ] + }, + ] + + cpp_typemaps += shared_typemaps + blink_cpp_typemaps = shared_typemaps + export_class_attribute_blink = "BLINK_PLATFORM_EXPORT" export_define_blink = "BLINK_PLATFORM_IMPLEMENTATION=1" export_header_blink = "third_party/blink/public/platform/web_common.h" } +source_set("shared_mojom_traits") { + sources = [ + "video_frame_metadata_mojom_traits.cc", + "video_frame_metadata_mojom_traits.h", + ] + + public_deps = [ + ":mojom_shared", + "//gpu/ipc/common:common", + "//gpu/ipc/common:mojom_traits", + "//media", + "//media/base/ipc:ipc", + "//mojo/public/mojom/base", + "//ui/gfx/geometry/mojom:mojom_traits", + ] +} + mojom("remoting_common") { sources = [ "remoting_common.mojom" ] } @@ -97,7 +299,10 @@ mojom("mirror_service_remoting") { mojom("remoting") { sources = [ "remoting.mojom" ] - public_deps = [ ":remoting_common" ] + public_deps = [ + ":mojom", + ":remoting_common", + ] } mojom("test_interfaces") { @@ -114,6 +319,7 @@ source_set("unit_tests") { "cdm_key_information_mojom_traits_unittest.cc", "video_decoder_config_mojom_traits_unittest.cc", "video_encode_accelerator_mojom_traits_unittest.cc", + "video_frame_metadata_mojom_traits_unittest.cc", "video_frame_mojom_traits_unittest.cc", ] diff --git a/chromium/media/mojo/mojom/content_decryption_module.mojom b/chromium/media/mojo/mojom/content_decryption_module.mojom index ca8d3080472..6cf18916b66 100644 --- a/chromium/media/mojo/mojom/content_decryption_module.mojom +++ b/chromium/media/mojo/mojom/content_decryption_module.mojom @@ -5,7 +5,6 @@ module media.mojom; import "media/mojo/mojom/decryptor.mojom"; -import "url/mojom/origin.mojom"; import "url/mojom/url.mojom"; // See media::EmeInitDataType. @@ -79,9 +78,7 @@ interface ContentDecryptionModule { // will be zero. Upon success, |cdm_id| will be non-zero and will later be // used to locate the CDM at the remote side. |decryptor| is the remote // Decryptor. - Initialize(string key_system, - url.mojom.Origin security_origin, - CdmConfig cdm_config) + Initialize(string key_system, CdmConfig cdm_config) => (CdmPromiseResult result, int32 cdm_id, pending_remote<Decryptor>? decryptor); diff --git a/chromium/media/mojo/mojom/frame_interface_factory.mojom b/chromium/media/mojo/mojom/frame_interface_factory.mojom index 49a153bd05f..6b56a3e06ca 100644 --- a/chromium/media/mojo/mojom/frame_interface_factory.mojom +++ b/chromium/media/mojo/mojom/frame_interface_factory.mojom @@ -7,6 +7,7 @@ module media.mojom; import "media/mojo/mojom/cdm_storage.mojom"; import "media/mojo/mojom/provision_fetcher.mojom"; import "mojo/public/mojom/base/generic_pending_receiver.mojom"; +import "url/mojom/origin.mojom"; // A factory for acquiring media mojo interfaces that are bound to a // RenderFrameHost. @@ -18,6 +19,10 @@ interface FrameInterfaceFactory { // CDM storage available. CreateCdmStorage(pending_receiver<CdmStorage> cdm_storage); + // Gets the origin of the frame associated with the CDM. + [Sync] + GetCdmOrigin() => (url.mojom.Origin cdm_origin); + // Binds a generic media frame-bound interface. This is to allow //content // embedders to provide additional interfaces. BindEmbedderReceiver(mojo_base.mojom.GenericPendingReceiver receiver); diff --git a/chromium/media/mojo/mojom/media_types.mojom b/chromium/media/mojo/mojom/media_types.mojom index 6b083bfe691..2696addb60e 100644 --- a/chromium/media/mojo/mojom/media_types.mojom +++ b/chromium/media/mojo/mojom/media_types.mojom @@ -8,6 +8,7 @@ import "gpu/ipc/common/mailbox_holder.mojom"; import "gpu/ipc/common/vulkan_ycbcr_info.mojom"; import "mojo/public/mojom/base/time.mojom"; import "mojo/public/mojom/base/values.mojom"; +import "mojo/public/mojom/base/unguessable_token.mojom"; import "ui/gfx/geometry/mojom/geometry.mojom"; import "ui/gfx/mojom/buffer_types.mojom"; import "ui/gfx/mojom/color_space.mojom"; @@ -65,8 +66,12 @@ enum VideoCodecProfile; enum VideoPixelFormat; // See media/base/video_transformation.h for descriptions. -[Native] -enum VideoRotation; +enum VideoRotation { + kVideoRotation0, + kVideoRotation90, + kVideoRotation180, + kVideoRotation270, +}; // See media/base/video_transformation.h for descriptions. struct VideoTransformation { @@ -255,6 +260,80 @@ struct AudioDataS16 { array<int16> data; }; +// See media/base/video_frame_metadata.h for a description of fields. +// TODO(crbug.com/657632): Remove |has_*| values and use nullable types. +struct VideoFrameMetadata { + bool allow_overlay; + + mojo_base.mojom.TimeTicks? capture_begin_time; + mojo_base.mojom.TimeTicks? capture_end_time; + + bool has_capture_counter; + int32 capture_counter; + + gfx.mojom.Rect? capture_update_rect; + + bool copy_required; + + bool end_of_stream; + + mojo_base.mojom.TimeDelta? frame_duration; + + bool has_frame_rate; + double frame_rate; + + bool interactive_content; + + mojo_base.mojom.TimeTicks? reference_time; + + bool has_resource_utilization; + double resource_utilization; + + bool read_lock_fences_enabled; + + bool has_rotation; + VideoRotation rotation; + + bool texture_owner; + + bool wants_promotion_hint; + + bool protected_video; + + bool hw_protected; + + mojo_base.mojom.UnguessableToken? overlay_plane_id; + + bool power_efficient; + + bool has_device_scale_factor; + double device_scale_factor; + + bool has_page_scale_factor; + double page_scale_factor; + + bool has_root_scroll_offset_x; + double root_scroll_offset_x; + + bool has_root_scroll_offset_y; + double root_scroll_offset_y; + + bool has_top_controls_visible_height; + double top_controls_visible_height; + + mojo_base.mojom.TimeTicks? decode_begin_time; + mojo_base.mojom.TimeTicks? decode_end_time; + + mojo_base.mojom.TimeDelta? processing_time; + + bool has_rtp_timestamp; + double rtp_timestamp; + + mojo_base.mojom.TimeTicks? receive_time; + + mojo_base.mojom.TimeDelta? wallclock_frame_duration; +}; + // This defines a mojo transport format for media::VideoFrame. struct VideoFrame { // Format of the frame. @@ -276,7 +355,7 @@ struct VideoFrame { VideoFrameData data; // Extra properties associated with the VideoFrame. - mojo_base.mojom.DictionaryValue metadata; + VideoFrameMetadata metadata; gfx.mojom.ColorSpace color_space; HDRMetadata? hdr_metadata; diff --git a/chromium/media/mojo/mojom/media_types.typemap b/chromium/media/mojo/mojom/media_types.typemap deleted file mode 100644 index 46c27d067cf..00000000000 --- a/chromium/media/mojo/mojom/media_types.typemap +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2016 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -mojom = "//media/mojo/mojom/media_types.mojom" - -public_headers = [ - "//media/base/audio_codecs.h", - "//media/base/buffering_state.h", - "//media/base/channel_layout.h", - "//media/base/container_names.h", - "//media/base/decode_status.h", - "//media/base/decrypt_config.h", - "//media/base/encryption_pattern.h", - "//media/base/encryption_scheme.h", - "//media/base/hdr_metadata.h", - "//media/base/media_log_record.h", - "//media/base/media_status.h", - "//media/base/output_device_info.h", - "//media/base/pipeline_status.h", - "//media/base/sample_format.h", - "//media/base/subsample_entry.h", - "//media/base/video_codecs.h", - "//media/base/video_transformation.h", - "//media/base/video_types.h", - "//media/base/waiting.h", - "//media/base/watch_time_keys.h", - "//media/base/status.h", - "//media/base/status_codes.h", -] - -traits_headers = [ - "//media/base/ipc/media_param_traits_macros.h", - "//media/mojo/mojom/video_transformation_mojom_traits.h", -] - -public_deps = [ - "//media", - "//media/base/ipc", -] - -sources = [ - "//media/mojo/mojom/video_transformation_mojom_traits.cc", - "//media/mojo/mojom/video_transformation_mojom_traits.h", -] - -type_mappings = [ - "media.mojom.AudioCodec=::media::AudioCodec", - "media.mojom.AudioCodecProfile=::media::AudioCodecProfile", - "media.mojom.BufferingState=::media::BufferingState", - "media.mojom.BufferingStateChangeReason=::media::BufferingStateChangeReason", - "media.mojom.ChannelLayout=::media::ChannelLayout", - "media.mojom.DecodeStatus=::media::DecodeStatus", - "media.mojom.EncryptionScheme=::media::EncryptionScheme", - "media.mojom.MediaContainerName=::media::container_names::MediaContainerName", - "media.mojom.MediaLogRecord=::media::MediaLogRecord", - "media.mojom.OutputDeviceStatus=::media::OutputDeviceStatus", - "media.mojom.PipelineStatus=::media::PipelineStatus", - "media.mojom.SampleFormat=::media::SampleFormat", - "media.mojom.SubsampleEntry=::media::SubsampleEntry", - "media.mojom.VideoCodec=::media::VideoCodec", - "media.mojom.VideoCodecProfile=::media::VideoCodecProfile", - "media.mojom.VideoPixelFormat=::media::VideoPixelFormat", - "media.mojom.VideoRotation=::media::VideoRotation", - "media.mojom.VideoTransformation=::media::VideoTransformation", - "media.mojom.WaitingReason=::media::WaitingReason", - "media.mojom.WatchTimeKey=::media::WatchTimeKey", - "media.mojom.MediaStatusState=::media::MediaStatus::State", - "media.mojom.StatusCode=::media::StatusCode", -] diff --git a/chromium/media/mojo/mojom/media_types_enum_mojom_traits.h b/chromium/media/mojo/mojom/media_types_enum_mojom_traits.h new file mode 100644 index 00000000000..d048fd01e4d --- /dev/null +++ b/chromium/media/mojo/mojom/media_types_enum_mojom_traits.h @@ -0,0 +1,63 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_MOJO_MOJOM_MEDIA_TYPES_ENUM_MOJOM_TRAITS_H_ +#define MEDIA_MOJO_MOJOM_MEDIA_TYPES_ENUM_MOJOM_TRAITS_H_ + +#include "base/notreached.h" +#include "media/base/video_transformation.h" +#include "media/mojo/mojom/media_types.mojom-shared.h" + +// Most enums have automatically generated traits, in media_types.mojom.h, due +// to their [native] attribute. This file defines traits for enums that are used +// in files that cannot directly include media_types.mojom.h. + +namespace mojo { + +template <> +struct EnumTraits<media::mojom::VideoRotation, ::media::VideoRotation> { + static media::mojom::VideoRotation ToMojom(::media::VideoRotation input) { + switch (input) { + case ::media::VideoRotation::VIDEO_ROTATION_0: + return media::mojom::VideoRotation::kVideoRotation0; + case ::media::VideoRotation::VIDEO_ROTATION_90: + return media::mojom::VideoRotation::kVideoRotation90; + case ::media::VideoRotation::VIDEO_ROTATION_180: + return media::mojom::VideoRotation::kVideoRotation180; + case ::media::VideoRotation::VIDEO_ROTATION_270: + return media::mojom::VideoRotation::kVideoRotation270; + } + + NOTREACHED(); + return static_cast<media::mojom::VideoRotation>(input); + } + + // Returning false results in deserialization failure and causes the + // message pipe receiving it to be disconnected. + static bool FromMojom(media::mojom::VideoRotation input, + media::VideoRotation* output) { + switch (input) { + case media::mojom::VideoRotation::kVideoRotation0: + *output = ::media::VideoRotation::VIDEO_ROTATION_0; + return true; + case media::mojom::VideoRotation::kVideoRotation90: + *output = ::media::VideoRotation::VIDEO_ROTATION_90; + return true; + case media::mojom::VideoRotation::kVideoRotation180: + *output = ::media::VideoRotation::VIDEO_ROTATION_180; + return true; + case media::mojom::VideoRotation::kVideoRotation270: + *output = ::media::VideoRotation::VIDEO_ROTATION_270; + return true; + } + + NOTREACHED(); + *output = static_cast<::media::VideoRotation>(input); + return true; + } +}; + +} // namespace mojo + +#endif // MEDIA_MOJO_MOJOM_MEDIA_TYPES_ENUM_MOJOM_TRAITS_H_ diff --git a/chromium/media/mojo/mojom/pipeline_status.typemap b/chromium/media/mojo/mojom/pipeline_status.typemap deleted file mode 100644 index be73386b42f..00000000000 --- a/chromium/media/mojo/mojom/pipeline_status.typemap +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2019 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -mojom = "//media/mojo/mojom/media_types.mojom" -public_headers = [ "//media/base/pipeline_status.h" ] -traits_headers = [ "//media/mojo/mojom/pipeline_status_mojom_traits.h" ] -type_mappings = [ - "media.mojom.PipelineStatistics=::media::PipelineStatistics", - "media.mojom.PipelineDecoderInfo=::media::PipelineDecoderInfo", -] diff --git a/chromium/media/mojo/mojom/remoting.mojom b/chromium/media/mojo/mojom/remoting.mojom index 6146cbd70d4..860eb94e022 100644 --- a/chromium/media/mojo/mojom/remoting.mojom +++ b/chromium/media/mojo/mojom/remoting.mojom @@ -4,7 +4,9 @@ module media.mojom; +import "media/mojo/mojom/media_types.mojom"; import "media/mojo/mojom/remoting_common.mojom"; +import "ui/gfx/geometry/mojom/geometry.mojom"; interface RemoterFactory { // Create a new Remoter associated with the given RemotingSource and bind it @@ -95,3 +97,59 @@ interface RemotingSource { // have caused remoting to end. OnStopped(RemotingStopReason reason); }; + +// Interface that is implemented by the host of RemotingSink and +// RemotingDataStreamReceiver. Remotee implementation would be implemented in +// the browser process in order to receive serialized RPC messages and frame +// data from the sender. +interface Remotee { + // Used by RemotingSink to notify Remotee that it is ready for remoting. + OnRemotingSinkReady(pending_remote<RemotingSink> sink); + + // Used by the RemotingSink to send a serialized RPC message to sender side. + // |message| is a serialized protobuf from src/media/remoting/proto. + SendMessageToSource(array<uint8> message); + + // Initialize the data pipe for RemotingDataStreamReceiver to allow Remotee to + // send frame data to it. + // Remoting media could be audio-only media, video-only media, or media has + // both audio and video. So, at least one of audio stream or video stream + // should be passed. + StartDataStreams( + pending_remote<RemotingDataStreamReceiver>? audio_stream, + pending_remote<RemotingDataStreamReceiver>? video_stream); + + // Used by RemotingSink to notify Remotee that FlushUntil is happening in + // order to not send NACK for the frames that are ignored. The implementation + // should also forward the |{audio/video}_frame_count| to FlushUntil() of + // {audio|video} streams which are RemotingDataStreamReceiver implementations. + OnFlushUntil(uint32 audio_frame_count, uint32 video_frame_count); + + // Used by RemotingSink to notify Remotee that VideoNaturalSizeChange is + // happening. + OnVideoNaturalSizeChange(gfx.mojom.Size size); +}; + +// Interface that is used to receive messages from the sender. +interface RemotingSink { + // Used by the Remotee to send a serialized RPC message to the sink. + // |message| is a serialized protobuf from src/media/remoting/proto. + OnMessageFromSource(array<uint8> message); +}; + +// Interface that is implemented by either an audio or a video demuxer stream at +// the receiver side to receive frame data. Passed to a Remotee which will send +// it frame data. +interface RemotingDataStreamReceiver { + // Used by the Remotee implementation to bind a data pipe to + // RemotingDataStreamReceiver. + InitializeDataPipe(handle<data_pipe_consumer> data_pipe); + + // Used by the Remotee implementation to send frame data to + // RemotingDataStreamReceiver. + ReceiveFrame(uint32 frame_count, DecoderBuffer buffer); + + // Used by the Remotee implementation to flush frames until the given frame + // count. + FlushUntil(uint32 frame_count); +}; diff --git a/chromium/media/mojo/mojom/speech_recognition_service.mojom b/chromium/media/mojo/mojom/speech_recognition_service.mojom index 6835c22c5ce..34dc344a8a7 100644 --- a/chromium/media/mojo/mojom/speech_recognition_service.mojom +++ b/chromium/media/mojo/mojom/speech_recognition_service.mojom @@ -5,15 +5,18 @@ module media.mojom; import "media/mojo/mojom/media_types.mojom"; +import "services/network/public/mojom/url_loader_factory.mojom"; // The main interface a client uses to interact with a speech recognition // service process. Every renderer can own one or more // Remote<SpeechRecognitionContext>, with the receiver bound through the -// BrowserInterfaceBroker. +// BrowserInterfaceBroker. Returns a flag indicating whether multichannel +// audio is supported by the speech recognition service. interface SpeechRecognitionContext { // Bind the recognizers to the speech recognition service. BindRecognizer(pending_receiver<SpeechRecognitionRecognizer> receiver, - pending_remote<SpeechRecognitionRecognizerClient> client); + pending_remote<SpeechRecognitionRecognizerClient> client) + => (bool is_multichannel_supported); }; // The main interface to a speech secognition service process. @@ -22,6 +25,23 @@ interface SpeechRecognitionContext { interface SpeechRecognitionService { // Bind the context to a new instance of the speech recognition. BindContext(pending_receiver<SpeechRecognitionContext> context); + + // Sets the URL loader factory used to create network requests. + SetUrlLoaderFactory( + pending_remote<network.mojom.URLLoaderFactory> url_loader_factory); + + // Binds the speech recognition service client used by the speech + // recognition service to send messages back to the client. + BindSpeechRecognitionServiceClient( + pending_remote<SpeechRecognitionServiceClient> client); +}; + +// The interface used to send messages from the speech recognition service +// back to the consumer of the service. +interface SpeechRecognitionServiceClient { + // Executed when the network service crashes, prompting the client to + // reset the URL loader factory. + OnNetworkServiceDisconnect(); }; // The interface used to pass raw audio from the renderer to the speech @@ -46,5 +66,9 @@ interface SpeechRecognitionRecognizerClient { // renderer. struct SpeechRecognitionResult { string transcription; + + // A flag indicating whether the result is final. If true, the result is + // locked in and the next result returned will not overlap with the previous + // final result. bool is_final; }; diff --git a/chromium/media/mojo/mojom/status.typemap b/chromium/media/mojo/mojom/status.typemap deleted file mode 100644 index 431610801b0..00000000000 --- a/chromium/media/mojo/mojom/status.typemap +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2020 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -mojom = "//media/mojo/mojom/media_types.mojom" - -public_headers = [ - "//media/base/status.h", - "//media/base/status_codes.h", -] - -traits_headers = [ "//media/mojo/mojom/status_mojom_traits.h" ] - -sources = [ "//media/mojo/mojom/status_mojom_traits.cc" ] - -public_deps = [ - "//base", - "//media", -] - -deps = [ "//media/base/ipc" ] - -# See media_types.typemap for enum mappings. -type_mappings = [ "media.mojom.Status=::media::Status" ] diff --git a/chromium/media/mojo/mojom/traits_test_service.mojom b/chromium/media/mojo/mojom/traits_test_service.mojom index 734581a6220..8e02156cb1f 100644 --- a/chromium/media/mojo/mojom/traits_test_service.mojom +++ b/chromium/media/mojo/mojom/traits_test_service.mojom @@ -10,3 +10,10 @@ interface TraitsTestService { [Sync] EchoVideoFrame(VideoFrame? f) => (VideoFrame? pass); }; + +// Test service to help with verifying VideoFrameMetadata traits. +interface VideoFrameMetadataTraitsTestService { + // Serializes and deserializes VideoFrameMedata. + [Sync] + EchoVideoFrameMetadata(VideoFrameMetadata vfm) => (VideoFrameMetadata pass); +}; diff --git a/chromium/media/mojo/mojom/typemaps.gni b/chromium/media/mojo/mojom/typemaps.gni index 44e4010102d..1bd18e7fc3d 100644 --- a/chromium/media/mojo/mojom/typemaps.gni +++ b/chromium/media/mojo/mojom/typemaps.gni @@ -13,15 +13,10 @@ typemaps = [ "//media/mojo/mojom/demuxer_stream.typemap", "//media/mojo/mojom/encryption_pattern.typemap", "//media/mojo/mojom/hdr_metadata.typemap", - "//media/mojo/mojom/media_types.typemap", - "//media/mojo/mojom/pipeline_status.typemap", - "//media/mojo/mojom/video_color_space.typemap", "//media/mojo/mojom/video_decoder.typemap", "//media/mojo/mojom/video_decoder_config.typemap", "//media/mojo/mojom/video_encode_accelerator.typemap", "//media/mojo/mojom/video_encoder_info.typemap", - "//media/mojo/mojom/video_frame.typemap", - "//media/mojo/mojom/status.typemap", ] if (enable_media_drm_storage) { diff --git a/chromium/media/mojo/mojom/video_color_space.typemap b/chromium/media/mojo/mojom/video_color_space.typemap deleted file mode 100644 index 12be2cb1cbf..00000000000 --- a/chromium/media/mojo/mojom/video_color_space.typemap +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -mojom = "//media/mojo/mojom/media_types.mojom" -public_headers = [ - "//media/base/video_color_space.h", - "//ui/gfx/color_space.h", -] -traits_headers = [ "//media/mojo/mojom/video_color_space_mojom_traits.h" ] -type_mappings = [ - "media.mojom.VideoColorSpace.PrimaryID=::media::VideoColorSpace::PrimaryID", - "media.mojom.VideoColorSpace.TransferID=::media::VideoColorSpace::TransferID", - "media.mojom.VideoColorSpace.MatrixID=::media::VideoColorSpace::MatrixID", - "media.mojom.VideoColorSpace.RangeID=::gfx::ColorSpace::RangeID", - "media.mojom.VideoColorSpace=::media::VideoColorSpace", -] diff --git a/chromium/media/mojo/mojom/video_frame.typemap b/chromium/media/mojo/mojom/video_frame.typemap deleted file mode 100644 index d465abcc3cc..00000000000 --- a/chromium/media/mojo/mojom/video_frame.typemap +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -mojom = "//media/mojo/mojom/media_types.mojom" - -# Additional headers required by any code which would depend on the mojom -# definition of media.mojom.VideoFrame now that the typemap is applied. Any -# headers required for the native target type definition should be listed here. -public_headers = [ - "//base/memory/ref_counted.h", - "//media/base/video_frame.h", -] - -# Headers which contain the relevant StructTraits specialization(s) for any -# type mappings described by this file. -traits_headers = [ "//media/mojo/mojom/video_frame_mojom_traits.h" ] - -sources = [ - "video_frame_mojom_traits.cc", -] - -# Target dependencies exposed by the public_headers and traits_headers. -public_deps = [ - "//base", - "//media", -] - -deps = [ - "//gpu/ipc/common:common", - "//gpu/ipc/common:mojom_traits", - "//media/base/ipc", - "//media/mojo/common:mojo_shared_buffer_video_frame", - "//ui/gfx/geometry/mojom:mojom_traits", -] - -type_mappings = [ "media.mojom.VideoFrame=::scoped_refptr<::media::VideoFrame>[nullable_is_same_type]" ] diff --git a/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits.cc b/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits.cc new file mode 100644 index 00000000000..0fe7306b64c --- /dev/null +++ b/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits.cc @@ -0,0 +1,86 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/mojo/mojom/video_frame_metadata_mojom_traits.h" + +#include <utility> + +#include "base/bind_helpers.h" +#include "base/logging.h" +#include "build/build_config.h" +#include "mojo/public/cpp/base/time_mojom_traits.h" +#include "mojo/public/cpp/base/unguessable_token_mojom_traits.h" + +namespace mojo { + +// Deserializes has_field and field into a base::Optional. +#define DESERIALIZE_INTO_OPT(field) \ + if (input.has_##field()) \ + output->field = input.field() + +#define READ_AND_ASSIGN_OPT(type, field, FieldInCamelCase) \ + base::Optional<type> field; \ + if (!input.Read##FieldInCamelCase(&field)) \ + return false; \ + \ + output->field = field + +// static +bool StructTraits<media::mojom::VideoFrameMetadataDataView, + media::VideoFrameMetadata>:: + Read(media::mojom::VideoFrameMetadataDataView input, + media::VideoFrameMetadata* output) { + // int. + DESERIALIZE_INTO_OPT(capture_counter); + + // bool. + output->allow_overlay = input.allow_overlay(); + output->copy_required = input.copy_required(); + output->end_of_stream = input.end_of_stream(); + output->texture_owner = input.texture_owner(); + output->wants_promotion_hint = input.wants_promotion_hint(); + output->protected_video = input.protected_video(); + output->hw_protected = input.hw_protected(); + output->power_efficient = input.power_efficient(); + output->read_lock_fences_enabled = input.read_lock_fences_enabled(); + output->interactive_content = input.interactive_content(); + + // double. + DESERIALIZE_INTO_OPT(device_scale_factor); + DESERIALIZE_INTO_OPT(page_scale_factor); + DESERIALIZE_INTO_OPT(root_scroll_offset_x); + DESERIALIZE_INTO_OPT(root_scroll_offset_y); + DESERIALIZE_INTO_OPT(top_controls_visible_height); + DESERIALIZE_INTO_OPT(resource_utilization); + DESERIALIZE_INTO_OPT(frame_rate); + DESERIALIZE_INTO_OPT(rtp_timestamp); + + if (input.has_rotation()) { + media::VideoRotation rotation; + if (!input.ReadRotation(&rotation)) + return false; + + output->rotation = rotation; + } + + READ_AND_ASSIGN_OPT(base::UnguessableToken, overlay_plane_id, OverlayPlaneId); + + READ_AND_ASSIGN_OPT(gfx::Rect, capture_update_rect, CaptureUpdateRect); + + READ_AND_ASSIGN_OPT(base::TimeTicks, receive_time, ReceiveTime); + READ_AND_ASSIGN_OPT(base::TimeTicks, capture_begin_time, CaptureBeginTime); + READ_AND_ASSIGN_OPT(base::TimeTicks, capture_end_time, CaptureEndTime); + READ_AND_ASSIGN_OPT(base::TimeTicks, decode_begin_time, DecodeBeginTime); + READ_AND_ASSIGN_OPT(base::TimeTicks, decode_end_time, DecodeEndTime); + READ_AND_ASSIGN_OPT(base::TimeTicks, reference_time, ReferenceTime); + + READ_AND_ASSIGN_OPT(base::TimeDelta, processing_time, ProcessingTime); + READ_AND_ASSIGN_OPT(base::TimeDelta, frame_duration, FrameDuration); + READ_AND_ASSIGN_OPT(base::TimeDelta, wallclock_frame_duration, + WallclockFrameDuration); + + return true; +} + +} // namespace mojo
\ No newline at end of file diff --git a/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits.h b/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits.h new file mode 100644 index 00000000000..8b07ef38e1e --- /dev/null +++ b/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits.h @@ -0,0 +1,148 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_MOJO_MOJOM_VIDEO_FRAME_METADATA_MOJOM_TRAITS_H_ +#define MEDIA_MOJO_MOJOM_VIDEO_FRAME_METADATA_MOJOM_TRAITS_H_ + +#include "base/memory/ref_counted.h" +#include "base/optional.h" +#include "media/base/ipc/media_param_traits_macros.h" +#include "media/base/video_frame_metadata.h" +#include "media/mojo/mojom/media_types.mojom-shared.h" +#include "media/mojo/mojom/media_types_enum_mojom_traits.h" +#include "mojo/public/cpp/bindings/struct_traits.h" +#include "ui/gfx/geometry/mojom/geometry_mojom_traits.h" + +namespace mojo { + +// Creates a has_foo() and a foo() to serialize a foo base::Optional<>. +#define GENERATE_OPT_SERIALIZATION(type, field, default_value) \ + static bool has_##field(const media::VideoFrameMetadata& input) { \ + return input.field.has_value(); \ + } \ + \ + static type field(const media::VideoFrameMetadata& input) { \ + return input.field.value_or(default_value); \ + } + +template <> +struct StructTraits<media::mojom::VideoFrameMetadataDataView, + media::VideoFrameMetadata> { + static bool allow_overlay(const media::VideoFrameMetadata& input) { + return input.allow_overlay; + } + + static bool copy_required(const media::VideoFrameMetadata& input) { + return input.copy_required; + } + + static bool end_of_stream(const media::VideoFrameMetadata& input) { + return input.end_of_stream; + } + + static bool texture_owner(const media::VideoFrameMetadata& input) { + return input.texture_owner; + } + + static bool wants_promotion_hint(const media::VideoFrameMetadata& input) { + return input.wants_promotion_hint; + } + + static bool protected_video(const media::VideoFrameMetadata& input) { + return input.protected_video; + } + + static bool hw_protected(const media::VideoFrameMetadata& input) { + return input.hw_protected; + } + + static bool power_efficient(const media::VideoFrameMetadata& input) { + return input.power_efficient; + } + + static bool read_lock_fences_enabled(const media::VideoFrameMetadata& input) { + return input.read_lock_fences_enabled; + } + + static bool interactive_content(const media::VideoFrameMetadata& input) { + return input.interactive_content; + } + + GENERATE_OPT_SERIALIZATION(int, capture_counter, 0) + + GENERATE_OPT_SERIALIZATION(media::VideoRotation, + rotation, + media::VideoRotation::VIDEO_ROTATION_0) + + GENERATE_OPT_SERIALIZATION(double, device_scale_factor, 0.0) + GENERATE_OPT_SERIALIZATION(double, page_scale_factor, 0.0) + GENERATE_OPT_SERIALIZATION(double, root_scroll_offset_x, 0.0) + GENERATE_OPT_SERIALIZATION(double, root_scroll_offset_y, 0.0) + GENERATE_OPT_SERIALIZATION(double, top_controls_visible_height, 0.0) + GENERATE_OPT_SERIALIZATION(double, resource_utilization, 0.0) + GENERATE_OPT_SERIALIZATION(double, frame_rate, 0.0) + GENERATE_OPT_SERIALIZATION(double, rtp_timestamp, 0.0) + + static base::Optional<gfx::Rect> capture_update_rect( + const media::VideoFrameMetadata& input) { + return input.capture_update_rect; + } + + static base::Optional<base::UnguessableToken> overlay_plane_id( + const media::VideoFrameMetadata& input) { + return input.overlay_plane_id; + } + + static base::Optional<base::TimeTicks> receive_time( + const media::VideoFrameMetadata& input) { + return input.receive_time; + } + + static base::Optional<base::TimeTicks> capture_begin_time( + const media::VideoFrameMetadata& input) { + return input.capture_begin_time; + } + + static base::Optional<base::TimeTicks> capture_end_time( + const media::VideoFrameMetadata& input) { + return input.capture_end_time; + } + + static base::Optional<base::TimeTicks> decode_begin_time( + const media::VideoFrameMetadata& input) { + return input.decode_begin_time; + } + + static base::Optional<base::TimeTicks> decode_end_time( + const media::VideoFrameMetadata& input) { + return input.decode_end_time; + } + + static base::Optional<base::TimeTicks> reference_time( + const media::VideoFrameMetadata& input) { + return input.reference_time; + } + + static base::Optional<base::TimeDelta> processing_time( + const media::VideoFrameMetadata& input) { + return input.processing_time; + } + + static base::Optional<base::TimeDelta> frame_duration( + const media::VideoFrameMetadata& input) { + return input.frame_duration; + } + + static base::Optional<base::TimeDelta> wallclock_frame_duration( + const media::VideoFrameMetadata& input) { + return input.wallclock_frame_duration; + } + + static bool Read(media::mojom::VideoFrameMetadataDataView input, + media::VideoFrameMetadata* output); +}; + +} // namespace mojo + +#endif // MEDIA_MOJO_MOJOM_VIDEO_FRAME_METADATA_MOJOM_TRAITS_H_ diff --git a/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits_unittest.cc b/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits_unittest.cc new file mode 100644 index 00000000000..4eaa51cc06e --- /dev/null +++ b/chromium/media/mojo/mojom/video_frame_metadata_mojom_traits_unittest.cc @@ -0,0 +1,193 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/mojo/mojom/video_frame_metadata_mojom_traits.h" + +#include "base/bind_helpers.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/test/task_environment.h" +#include "build/build_config.h" +#include "media/mojo/mojom/traits_test_service.mojom.h" +#include "mojo/public/cpp/bindings/receiver_set.h" +#include "mojo/public/cpp/bindings/remote.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gfx/geometry/rect.h" + +namespace media { + +namespace { + +class VideoFrameMetadataStructTraitsTest + : public testing::Test, + public media::mojom::VideoFrameMetadataTraitsTestService { + public: + VideoFrameMetadataStructTraitsTest() = default; + + protected: + mojo::Remote<mojom::VideoFrameMetadataTraitsTestService> + GetTraitsTestRemote() { + mojo::Remote<mojom::VideoFrameMetadataTraitsTestService> remote; + traits_test_receivers_.Add(this, remote.BindNewPipeAndPassReceiver()); + return remote; + } + + bool RoundTrip(const VideoFrameMetadata& in, VideoFrameMetadata* out) { + mojo::Remote<mojom::VideoFrameMetadataTraitsTestService> remote = + GetTraitsTestRemote(); + return remote->EchoVideoFrameMetadata(in, out); + } + + private: + void EchoVideoFrameMetadata( + const VideoFrameMetadata& vfm, + EchoVideoFrameMetadataCallback callback) override { + std::move(callback).Run(vfm); + } + + base::test::TaskEnvironment task_environment_; + mojo::ReceiverSet<VideoFrameMetadataTraitsTestService> traits_test_receivers_; + + DISALLOW_COPY_AND_ASSIGN(VideoFrameMetadataStructTraitsTest); +}; + +} // namespace + +TEST_F(VideoFrameMetadataStructTraitsTest, EmptyMetadata) { + VideoFrameMetadata metadata_in; + VideoFrameMetadata metadata_out; + + ASSERT_TRUE(RoundTrip(metadata_in, &metadata_out)); + + EXPECT_FALSE(metadata_out.capture_counter.has_value()); + EXPECT_FALSE(metadata_out.capture_update_rect.has_value()); + EXPECT_FALSE(metadata_out.rotation.has_value()); + EXPECT_FALSE(metadata_out.allow_overlay); + EXPECT_FALSE(metadata_out.copy_required); + EXPECT_FALSE(metadata_out.end_of_stream); + EXPECT_FALSE(metadata_out.texture_owner); + EXPECT_FALSE(metadata_out.wants_promotion_hint); + EXPECT_FALSE(metadata_out.protected_video); + EXPECT_FALSE(metadata_out.hw_protected); + EXPECT_FALSE(metadata_out.power_efficient); + EXPECT_FALSE(metadata_out.read_lock_fences_enabled); + EXPECT_FALSE(metadata_out.interactive_content); + EXPECT_FALSE(metadata_out.overlay_plane_id.has_value()); + EXPECT_FALSE(metadata_out.device_scale_factor.has_value()); + EXPECT_FALSE(metadata_out.page_scale_factor.has_value()); + EXPECT_FALSE(metadata_out.root_scroll_offset_x.has_value()); + EXPECT_FALSE(metadata_out.root_scroll_offset_y.has_value()); + EXPECT_FALSE(metadata_out.top_controls_visible_height.has_value()); + EXPECT_FALSE(metadata_out.resource_utilization.has_value()); + EXPECT_FALSE(metadata_out.frame_rate.has_value()); + EXPECT_FALSE(metadata_out.rtp_timestamp.has_value()); + EXPECT_FALSE(metadata_out.receive_time.has_value()); + EXPECT_FALSE(metadata_out.capture_begin_time.has_value()); + EXPECT_FALSE(metadata_out.capture_end_time.has_value()); + EXPECT_FALSE(metadata_out.decode_begin_time.has_value()); + EXPECT_FALSE(metadata_out.decode_end_time.has_value()); + EXPECT_FALSE(metadata_out.reference_time.has_value()); + EXPECT_FALSE(metadata_out.processing_time.has_value()); + EXPECT_FALSE(metadata_out.frame_duration.has_value()); + EXPECT_FALSE(metadata_out.wallclock_frame_duration.has_value()); +} + +TEST_F(VideoFrameMetadataStructTraitsTest, ValidMetadata) { + // Assign a non-default, distinct (when possible), value to all fields, and + // make sure values are preserved across serialization. + VideoFrameMetadata metadata_in; + + // ints + metadata_in.capture_counter = 123; + + // gfx::Rects + metadata_in.capture_update_rect = gfx::Rect(12, 34, 360, 480); + + // media::VideoRotations + metadata_in.rotation = media::VideoRotation::VIDEO_ROTATION_90; + + // bools + metadata_in.allow_overlay = true; + metadata_in.copy_required = true; + metadata_in.end_of_stream = true; + metadata_in.texture_owner = true; + metadata_in.wants_promotion_hint = true; + metadata_in.protected_video = true; + metadata_in.hw_protected = true; + metadata_in.power_efficient = true; + metadata_in.read_lock_fences_enabled = true; + metadata_in.interactive_content = true; + + // base::UnguessableTokens + metadata_in.overlay_plane_id = base::UnguessableToken::Create(); + + // doubles + metadata_in.device_scale_factor = 2.0; + metadata_in.page_scale_factor = 2.1; + metadata_in.root_scroll_offset_x = 100.2; + metadata_in.root_scroll_offset_y = 200.1; + metadata_in.top_controls_visible_height = 25.5; + metadata_in.resource_utilization = 95.8; + metadata_in.frame_rate = 29.94; + metadata_in.rtp_timestamp = 1.0; + + // base::TimeTicks + base::TimeTicks now = base::TimeTicks::Now(); + metadata_in.receive_time = now + base::TimeDelta::FromMilliseconds(10); + metadata_in.capture_begin_time = now + base::TimeDelta::FromMilliseconds(20); + metadata_in.capture_end_time = now + base::TimeDelta::FromMilliseconds(30); + metadata_in.decode_begin_time = now + base::TimeDelta::FromMilliseconds(40); + metadata_in.decode_end_time = now + base::TimeDelta::FromMilliseconds(50); + metadata_in.reference_time = now + base::TimeDelta::FromMilliseconds(60); + + // base::TimeDeltas + metadata_in.processing_time = base::TimeDelta::FromMilliseconds(500); + metadata_in.frame_duration = base::TimeDelta::FromMilliseconds(16); + metadata_in.wallclock_frame_duration = base::TimeDelta::FromMilliseconds(17); + + VideoFrameMetadata metadata_out; + + ASSERT_TRUE(RoundTrip(metadata_in, &metadata_out)); + + EXPECT_EQ(metadata_in.capture_counter, metadata_out.capture_counter); + EXPECT_EQ(metadata_in.capture_update_rect, metadata_out.capture_update_rect); + EXPECT_EQ(metadata_in.rotation, metadata_out.rotation); + EXPECT_EQ(metadata_in.allow_overlay, metadata_out.allow_overlay); + EXPECT_EQ(metadata_in.copy_required, metadata_out.copy_required); + EXPECT_EQ(metadata_in.end_of_stream, metadata_out.end_of_stream); + EXPECT_EQ(metadata_in.texture_owner, metadata_out.texture_owner); + EXPECT_EQ(metadata_in.wants_promotion_hint, + metadata_out.wants_promotion_hint); + EXPECT_EQ(metadata_in.protected_video, metadata_out.protected_video); + EXPECT_EQ(metadata_in.hw_protected, metadata_out.hw_protected); + EXPECT_EQ(metadata_in.power_efficient, metadata_out.power_efficient); + EXPECT_EQ(metadata_in.read_lock_fences_enabled, + metadata_out.read_lock_fences_enabled); + EXPECT_EQ(metadata_in.interactive_content, metadata_out.interactive_content); + EXPECT_EQ(metadata_in.overlay_plane_id, metadata_out.overlay_plane_id); + EXPECT_EQ(metadata_in.device_scale_factor, metadata_out.device_scale_factor); + EXPECT_EQ(metadata_in.page_scale_factor, metadata_out.page_scale_factor); + EXPECT_EQ(metadata_in.root_scroll_offset_x, + metadata_out.root_scroll_offset_x); + EXPECT_EQ(metadata_in.root_scroll_offset_y, + metadata_out.root_scroll_offset_y); + EXPECT_EQ(metadata_in.top_controls_visible_height, + metadata_out.top_controls_visible_height); + EXPECT_EQ(metadata_in.resource_utilization, + metadata_out.resource_utilization); + EXPECT_EQ(metadata_in.frame_rate, metadata_out.frame_rate); + EXPECT_EQ(metadata_in.rtp_timestamp, metadata_out.rtp_timestamp); + EXPECT_EQ(metadata_in.receive_time, metadata_out.receive_time); + EXPECT_EQ(metadata_in.capture_begin_time, metadata_out.capture_begin_time); + EXPECT_EQ(metadata_in.capture_end_time, metadata_out.capture_end_time); + EXPECT_EQ(metadata_in.decode_begin_time, metadata_out.decode_begin_time); + EXPECT_EQ(metadata_in.decode_end_time, metadata_out.decode_end_time); + EXPECT_EQ(metadata_in.reference_time, metadata_out.reference_time); + EXPECT_EQ(metadata_in.processing_time, metadata_out.processing_time); + EXPECT_EQ(metadata_in.frame_duration, metadata_out.frame_duration); + EXPECT_EQ(metadata_in.wallclock_frame_duration, + metadata_out.wallclock_frame_duration); +} + +} // namespace media diff --git a/chromium/media/mojo/mojom/video_frame_mojom_traits.cc b/chromium/media/mojo/mojom/video_frame_mojom_traits.cc index b2c93afe712..0f07949191d 100644 --- a/chromium/media/mojo/mojom/video_frame_mojom_traits.cc +++ b/chromium/media/mojo/mojom/video_frame_mojom_traits.cc @@ -15,8 +15,8 @@ #include "media/base/format_utils.h" #include "media/mojo/common/mojo_shared_buffer_video_frame.h" #include "media/mojo/mojom/hdr_metadata_mojom_traits.h" +#include "media/mojo/mojom/video_frame_metadata_mojom_traits.h" #include "mojo/public/cpp/base/time_mojom_traits.h" -#include "mojo/public/cpp/base/values_mojom_traits.h" #include "mojo/public/cpp/system/handle.h" #include "ui/gfx/mojom/buffer_types_mojom_traits.h" #include "ui/gfx/mojom/color_space_mojom_traits.h" @@ -31,7 +31,7 @@ namespace { media::mojom::VideoFrameDataPtr MakeVideoFrameData( const media::VideoFrame* input) { - if (input->metadata()->IsTrue(media::VideoFrameMetadata::END_OF_STREAM)) { + if (input->metadata()->end_of_stream) { return media::mojom::VideoFrameData::NewEosData( media::mojom::EosVideoFrameData::New()); } @@ -275,11 +275,11 @@ bool StructTraits<media::mojom::VideoFrameDataView, if (!frame) return false; - base::Value metadata; + media::VideoFrameMetadata metadata; if (!input.ReadMetadata(&metadata)) return false; - frame->metadata()->MergeInternalValuesFrom(metadata); + frame->set_metadata(metadata); gfx::ColorSpace color_space; if (!input.ReadColorSpace(&color_space)) diff --git a/chromium/media/mojo/mojom/video_frame_mojom_traits.h b/chromium/media/mojo/mojom/video_frame_mojom_traits.h index c064e76b79a..461bd9c6d3b 100644 --- a/chromium/media/mojo/mojom/video_frame_mojom_traits.h +++ b/chromium/media/mojo/mojom/video_frame_mojom_traits.h @@ -75,9 +75,11 @@ struct StructTraits<media::mojom::VideoFrameDataView, static media::mojom::VideoFrameDataPtr data( const scoped_refptr<media::VideoFrame>& input); - static const base::Value& metadata( + // TODO(https://crbug.com/1096727): Change VideoFrame::Metadata() to return a + // const &. + static const media::VideoFrameMetadata& metadata( const scoped_refptr<media::VideoFrame>& input) { - return input->metadata()->GetInternalValues(); + return *(input->metadata()); } static bool Read(media::mojom::VideoFrameDataView input, diff --git a/chromium/media/mojo/mojom/video_frame_mojom_traits_unittest.cc b/chromium/media/mojo/mojom/video_frame_mojom_traits_unittest.cc index 329fbcb7cc2..aaf290e2ddb 100644 --- a/chromium/media/mojo/mojom/video_frame_mojom_traits_unittest.cc +++ b/chromium/media/mojo/mojom/video_frame_mojom_traits_unittest.cc @@ -77,7 +77,7 @@ TEST_F(VideoFrameStructTraitsTest, EOS) { ASSERT_TRUE(RoundTrip(&frame)); ASSERT_TRUE(frame); - EXPECT_TRUE(frame->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)); + EXPECT_TRUE(frame->metadata()->end_of_stream); } TEST_F(VideoFrameStructTraitsTest, MojoSharedBufferVideoFrame) { @@ -86,15 +86,12 @@ TEST_F(VideoFrameStructTraitsTest, MojoSharedBufferVideoFrame) { scoped_refptr<VideoFrame> frame = MojoSharedBufferVideoFrame::CreateDefaultForTesting( format, gfx::Size(100, 100), base::TimeDelta::FromSeconds(100)); - frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE, 42.0); + frame->metadata()->frame_rate = 42.0; ASSERT_TRUE(RoundTrip(&frame)); ASSERT_TRUE(frame); - EXPECT_FALSE(frame->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)); - double frame_rate = 0.0; - EXPECT_TRUE(frame->metadata()->GetDouble(VideoFrameMetadata::FRAME_RATE, - &frame_rate)); - EXPECT_EQ(frame_rate, 42.0); + EXPECT_FALSE(frame->metadata()->end_of_stream); + EXPECT_EQ(*frame->metadata()->frame_rate, 42.0); EXPECT_EQ(frame->coded_size(), gfx::Size(100, 100)); EXPECT_EQ(frame->timestamp(), base::TimeDelta::FromSeconds(100)); @@ -129,7 +126,7 @@ TEST_F(VideoFrameStructTraitsTest, DmabufVideoFrame) { ASSERT_TRUE(RoundTrip(&frame)); ASSERT_TRUE(frame); - EXPECT_FALSE(frame->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)); + EXPECT_FALSE(frame->metadata()->end_of_stream); EXPECT_EQ(frame->format(), PIXEL_FORMAT_NV12); EXPECT_EQ(frame->coded_size(), gfx::Size(1280, 720)); EXPECT_EQ(frame->visible_rect(), gfx::Rect(0, 0, 1280, 720)); @@ -151,7 +148,7 @@ TEST_F(VideoFrameStructTraitsTest, MailboxVideoFrame) { ASSERT_TRUE(RoundTrip(&frame)); ASSERT_TRUE(frame); - EXPECT_FALSE(frame->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)); + EXPECT_FALSE(frame->metadata()->end_of_stream); EXPECT_EQ(frame->format(), PIXEL_FORMAT_ARGB); EXPECT_EQ(frame->coded_size(), gfx::Size(100, 100)); EXPECT_EQ(frame->visible_rect(), gfx::Rect(10, 10, 80, 80)); @@ -185,7 +182,7 @@ TEST_F(VideoFrameStructTraitsTest, GpuMemoryBufferVideoFrame) { ASSERT_TRUE(frame); ASSERT_EQ(frame->storage_type(), VideoFrame::STORAGE_GPU_MEMORY_BUFFER); EXPECT_TRUE(frame->HasGpuMemoryBuffer()); - EXPECT_FALSE(frame->metadata()->IsTrue(VideoFrameMetadata::END_OF_STREAM)); + EXPECT_FALSE(frame->metadata()->end_of_stream); EXPECT_EQ(frame->format(), PIXEL_FORMAT_NV12); EXPECT_EQ(frame->coded_size(), coded_size); EXPECT_EQ(frame->visible_rect(), visible_rect); diff --git a/chromium/media/mojo/mojom/video_transformation_mojom_traits.h b/chromium/media/mojo/mojom/video_transformation_mojom_traits.h index f788f6fc6a0..11ddd8126c9 100644 --- a/chromium/media/mojo/mojom/video_transformation_mojom_traits.h +++ b/chromium/media/mojo/mojom/video_transformation_mojom_traits.h @@ -8,6 +8,7 @@ #include "media/base/ipc/media_param_traits.h" #include "media/base/video_transformation.h" #include "media/mojo/mojom/media_types.mojom.h" +#include "media/mojo/mojom/media_types_enum_mojom_traits.h" namespace mojo { diff --git a/chromium/media/mojo/services/BUILD.gn b/chromium/media/mojo/services/BUILD.gn index 3f39a35e113..f91fa57a6bf 100644 --- a/chromium/media/mojo/services/BUILD.gn +++ b/chromium/media/mojo/services/BUILD.gn @@ -56,6 +56,8 @@ jumbo_component("services") { "mojo_video_encode_accelerator_provider.h", "mojo_video_encode_accelerator_service.cc", "mojo_video_encode_accelerator_service.h", + "playback_events_recorder.cc", + "playback_events_recorder.h", "test_mojo_media_client.cc", "test_mojo_media_client.h", "video_decode_perf_history.cc", @@ -109,10 +111,6 @@ jumbo_component("services") { ] } - if (is_fuchsia) { - deps += [ "//media/fuchsia/metrics" ] - } - if (enable_media_drm_storage) { sources += [ "mojo_media_drm_storage.cc", @@ -155,6 +153,7 @@ source_set("unit_tests") { "mojo_audio_output_stream_provider_unittest.cc", "mojo_audio_output_stream_unittest.cc", "mojo_video_encode_accelerator_service_unittest.cc", + "playback_events_recorder_test.cc", "test_helpers.cc", "test_helpers.h", "video_decode_perf_history_unittest.cc", diff --git a/chromium/media/mojo/services/cdm_service_unittest.cc b/chromium/media/mojo/services/cdm_service_unittest.cc index 02aec3d22e6..b98189d7b2b 100644 --- a/chromium/media/mojo/services/cdm_service_unittest.cc +++ b/chromium/media/mojo/services/cdm_service_unittest.cc @@ -17,7 +17,6 @@ #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" -#include "url/origin.h" namespace media { @@ -33,7 +32,6 @@ MATCHER_P(MatchesResult, success, "") { const char kClearKeyKeySystem[] = "org.w3.clearkey"; const char kInvalidKeySystem[] = "invalid.key.system"; -const char kSecurityOrigin[] = "https://foo.com"; class MockCdmServiceClient : public media::CdmService::Client { public: @@ -99,10 +97,9 @@ class CdmServiceTest : public testing::Test { &CdmServiceTest::CdmConnectionClosed, base::Unretained(this))); EXPECT_CALL(*this, OnCdmInitialized(MatchesResult(expected_result), _, _)) .WillOnce(InvokeWithoutArgs(&run_loop, &base::RunLoop::Quit)); - cdm_remote_->Initialize( - key_system, url::Origin::Create(GURL(kSecurityOrigin)), CdmConfig(), - base::BindOnce(&CdmServiceTest::OnCdmInitialized, - base::Unretained(this))); + cdm_remote_->Initialize(key_system, CdmConfig(), + base::BindOnce(&CdmServiceTest::OnCdmInitialized, + base::Unretained(this))); run_loop.Run(); } diff --git a/chromium/media/mojo/services/deferred_destroy_unique_receiver_set.h b/chromium/media/mojo/services/deferred_destroy_unique_receiver_set.h index 5e974fcf489..e7801c90625 100644 --- a/chromium/media/mojo/services/deferred_destroy_unique_receiver_set.h +++ b/chromium/media/mojo/services/deferred_destroy_unique_receiver_set.h @@ -11,6 +11,7 @@ #include <memory> #include "base/bind.h" +#include "base/logging.h" #include "base/macros.h" #include "base/memory/ptr_util.h" #include "base/memory/weak_ptr.h" diff --git a/chromium/media/mojo/services/gpu_mojo_media_client.cc b/chromium/media/mojo/services/gpu_mojo_media_client.cc index 293591767dc..16a146b921c 100644 --- a/chromium/media/mojo/services/gpu_mojo_media_client.cc +++ b/chromium/media/mojo/services/gpu_mojo_media_client.cc @@ -243,7 +243,7 @@ std::unique_ptr<VideoDecoder> GpuMojoMediaClient::CreateVideoDecoder( // requiring the provider to set |is_vulkan| in the ImageRecord. auto frame_info_helper = FrameInfoHelper::Create(gpu_task_runner_, std::move(get_stub_cb)); - video_decoder = std::make_unique<MediaCodecVideoDecoder>( + video_decoder = MediaCodecVideoDecoder::Create( gpu_preferences_, gpu_feature_info_, media_log->Clone(), DeviceInfo::GetInstance(), CodecAllocator::GetInstance(gpu_task_runner_), @@ -269,7 +269,7 @@ std::unique_ptr<VideoDecoder> GpuMojoMediaClient::CreateVideoDecoder( command_buffer_id->route_id)); video_decoder = ChromeosVideoDecoderFactory::Create( task_runner, std::move(frame_pool), std::move(frame_converter), - gpu_memory_buffer_factory_); + media_log->Clone()); } else { video_decoder = VdaVideoDecoder::Create( task_runner, gpu_task_runner_, media_log->Clone(), diff --git a/chromium/media/mojo/services/media_metrics_provider.cc b/chromium/media/mojo/services/media_metrics_provider.cc index e0920fce86b..3b641080dde 100644 --- a/chromium/media/mojo/services/media_metrics_provider.cc +++ b/chromium/media/mojo/services/media_metrics_provider.cc @@ -11,6 +11,7 @@ #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" #include "build/build_config.h" +#include "build/chromecast_buildflags.h" #include "media/learning/mojo/mojo_learning_task_controller_service.h" #include "media/mojo/services/video_decode_stats_recorder.h" #include "media/mojo/services/watch_time_recorder.h" @@ -22,9 +23,9 @@ #include "media/filters/decrypting_video_decoder.h" #endif // !defined(OS_ANDROID) -#if defined(OS_FUCHSIA) -#include "media/fuchsia/metrics/fuchsia_playback_events_recorder.h" -#endif // defined(OS_FUCHSIA) +#if defined(OS_FUCHSIA) || (BUILDFLAG(IS_CHROMECAST) && defined(OS_ANDROID)) +#include "media/mojo/services/playback_events_recorder.h" +#endif namespace media { @@ -50,7 +51,7 @@ MediaMetricsProvider::MediaMetricsProvider( source_id_(source_id), origin_(origin), save_cb_(std::move(save_cb)), - learning_session_cb_(learning_session_cb), + learning_session_cb_(std::move(learning_session_cb)), record_playback_cb_(std::move(record_playback_cb)), uma_info_(is_incognito == BrowsingMode::kIncognito) {} @@ -89,37 +90,31 @@ std::string MediaMetricsProvider::GetUMANameForAVStream( const PipelineInfo& player_info) { constexpr char kPipelineUmaPrefix[] = "Media.PipelineStatus.AudioVideo."; std::string uma_name = kPipelineUmaPrefix; - if (player_info.video_codec == kCodecVP8) { + if (player_info.video_codec == kCodecVP8) uma_name += "VP8."; - } else if (player_info.video_codec == kCodecVP9) { + else if (player_info.video_codec == kCodecVP9) uma_name += "VP9."; - } else if (player_info.video_codec == kCodecH264) { + else if (player_info.video_codec == kCodecH264) uma_name += "H264."; - } else if (player_info.video_codec == kCodecAV1) { + else if (player_info.video_codec == kCodecAV1) uma_name += "AV1."; - } else { + else return uma_name + "Other"; - } #if !defined(OS_ANDROID) if (player_info.video_pipeline_info.decoder_name == - media::DecryptingVideoDecoder::kDecoderName) { + DecryptingVideoDecoder::kDecoderName) { return uma_name + "DVD"; } #endif - if (player_info.video_pipeline_info.has_decrypting_demuxer_stream) { + if (player_info.video_pipeline_info.has_decrypting_demuxer_stream) uma_name += "DDS."; - } // Note that HW essentially means 'platform' anyway. MediaCodec has been // reported as HW forever, regardless of the underlying platform // implementation. - if (player_info.video_pipeline_info.is_platform_decoder) { - uma_name += "HW"; - } else { - uma_name += "SW"; - } + uma_name += player_info.video_pipeline_info.is_platform_decoder ? "HW" : "SW"; return uma_name; } @@ -127,22 +122,22 @@ void MediaMetricsProvider::ReportPipelineUMA() { if (uma_info_.has_video && uma_info_.has_audio) { base::UmaHistogramExactLinear(GetUMANameForAVStream(uma_info_), uma_info_.last_pipeline_status, - media::PIPELINE_STATUS_MAX + 1); + PIPELINE_STATUS_MAX + 1); } else if (uma_info_.has_audio) { base::UmaHistogramExactLinear("Media.PipelineStatus.AudioOnly", uma_info_.last_pipeline_status, - media::PIPELINE_STATUS_MAX + 1); + PIPELINE_STATUS_MAX + 1); } else if (uma_info_.has_video) { base::UmaHistogramExactLinear("Media.PipelineStatus.VideoOnly", uma_info_.last_pipeline_status, - media::PIPELINE_STATUS_MAX + 1); + PIPELINE_STATUS_MAX + 1); } else { // Note: This metric can be recorded as a result of normal operation with // Media Source Extensions. If a site creates a MediaSource object but never // creates a source buffer or appends data, PIPELINE_OK will be recorded. base::UmaHistogramExactLinear("Media.PipelineStatus.Unsupported", uma_info_.last_pipeline_status, - media::PIPELINE_STATUS_MAX + 1); + PIPELINE_STATUS_MAX + 1); } // Report whether video decoder fallback happened, but only if a video decoder @@ -154,15 +149,13 @@ void MediaMetricsProvider::ReportPipelineUMA() { // Report whether this player ever saw a playback event. Used to measure the // effectiveness of efforts to reduce loaded-but-never-used players. - if (uma_info_.has_reached_have_enough) { + if (uma_info_.has_reached_have_enough) base::UmaHistogramBoolean("Media.HasEverPlayed", uma_info_.has_ever_played); - } // Report whether an encrypted playback is in incognito window, excluding // never-used players. - if (uma_info_.is_eme && uma_info_.has_ever_played) { + if (uma_info_.is_eme && uma_info_.has_ever_played) base::UmaHistogramBoolean("Media.EME.IsIncognito", uma_info_.is_incognito); - } } // static @@ -178,7 +171,8 @@ void MediaMetricsProvider::Create( mojo::MakeSelfOwnedReceiver( std::make_unique<MediaMetricsProvider>( is_incognito, is_top_frame, get_source_id_cb.Run(), - get_origin_cb.Run(), std::move(save_cb), learning_session_cb, + get_origin_cb.Run(), std::move(save_cb), + std::move(learning_session_cb), std::move(get_record_playback_cb).Run()), std::move(receiver)); } @@ -296,15 +290,14 @@ void MediaMetricsProvider::AcquireVideoDecodeStatsRecorder( void MediaMetricsProvider::AcquirePlaybackEventsRecorder( mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) { -#if defined(OS_FUCHSIA) - FuchsiaPlaybackEventsRecorder::Create(std::move(receiver)); +#if defined(OS_FUCHSIA) || (BUILDFLAG(IS_CHROMECAST) && defined(OS_ANDROID)) + PlaybackEventsRecorder::Create(std::move(receiver)); #endif } void MediaMetricsProvider::AcquireLearningTaskController( const std::string& taskName, - mojo::PendingReceiver<media::learning::mojom::LearningTaskController> - receiver) { + mojo::PendingReceiver<learning::mojom::LearningTaskController> receiver) { learning::LearningSession* session = learning_session_cb_.Run(); if (!session) { DVLOG(3) << __func__ << " Ignoring request, unable to get LearningSession."; diff --git a/chromium/media/mojo/services/media_metrics_provider.h b/chromium/media/mojo/services/media_metrics_provider.h index c9857f2c771..ef0ce4b523a 100644 --- a/chromium/media/mojo/services/media_metrics_provider.h +++ b/chromium/media/mojo/services/media_metrics_provider.h @@ -125,8 +125,8 @@ class MEDIA_MOJO_EXPORT MediaMetricsProvider mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) override; void AcquireLearningTaskController( const std::string& taskName, - mojo::PendingReceiver<media::learning::mojom::LearningTaskController> - receiver) override; + mojo::PendingReceiver<learning::mojom::LearningTaskController> receiver) + override; void ReportPipelineUMA(); std::string GetUMANameForAVStream(const PipelineInfo& player_info); diff --git a/chromium/media/mojo/services/media_service_unittest.cc b/chromium/media/mojo/services/media_service_unittest.cc index ef5f39687a0..085efa69c07 100644 --- a/chromium/media/mojo/services/media_service_unittest.cc +++ b/chromium/media/mojo/services/media_service_unittest.cc @@ -11,8 +11,8 @@ #include "base/callback.h" #include "base/macros.h" #include "base/run_loop.h" -#include "base/task/post_task.h" #include "base/test/task_environment.h" +#include "base/threading/thread_task_runner_handle.h" #include "build/build_config.h" #include "media/base/cdm_config.h" #include "media/base/mock_filters.h" @@ -35,7 +35,6 @@ #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" #include "url/gurl.h" -#include "url/origin.h" namespace media { @@ -59,8 +58,6 @@ const char kClearKeyKeySystem[] = "org.w3.clearkey"; const char kInvalidKeySystem[] = "invalid.key.system"; #endif -const char kSecurityOrigin[] = "https://foo.com"; - class MockRendererClient : public mojom::RendererClient { public: MockRendererClient() = default; @@ -90,7 +87,8 @@ class MockRendererClient : public mojom::RendererClient { }; ACTION_P(QuitLoop, run_loop) { - base::PostTask(FROM_HERE, run_loop->QuitClosure()); + base::ThreadTaskRunnerHandle::Get()->PostTask(FROM_HERE, + run_loop->QuitClosure()); } // Tests MediaService using TestMojoMediaClient, which supports CDM creation @@ -139,8 +137,7 @@ class MediaServiceTest : public testing::Test { // cdm_id" out and then call DoAll. EXPECT_CALL(*this, OnCdmInitialized(MatchesResult(expected_result), _, _)) .WillOnce(WithArg<1>(DoAll(SaveArg<0>(&cdm_id), QuitLoop(&run_loop)))); - cdm_->Initialize(key_system, url::Origin::Create(GURL(kSecurityOrigin)), - CdmConfig(), + cdm_->Initialize(key_system, CdmConfig(), base::BindOnce(&MediaServiceTest::OnCdmInitialized, base::Unretained(this))); run_loop.Run(); diff --git a/chromium/media/mojo/services/mojo_audio_decoder_service.h b/chromium/media/mojo/services/mojo_audio_decoder_service.h index c3ec39b1c1e..b51beb83992 100644 --- a/chromium/media/mojo/services/mojo_audio_decoder_service.h +++ b/chromium/media/mojo/services/mojo_audio_decoder_service.h @@ -20,7 +20,6 @@ namespace media { -class CdmContextRef; class MojoCdmServiceContext; class MojoDecoderBufferReader; diff --git a/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc b/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc index 04fa66ef823..4bc9f34f0a3 100644 --- a/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc +++ b/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc @@ -7,15 +7,16 @@ #include <utility> #include "base/bind.h" +#include "base/bind_helpers.h" #include "base/run_loop.h" #include "base/test/mock_callback.h" #include "base/test/task_environment.h" #include "build/build_config.h" #include "media/audio/audio_output_delegate.h" #include "media/base/audio_parameters.h" -#include "mojo/core/embedder/embedder.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/remote.h" +#include "mojo/public/cpp/system/functions.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -75,7 +76,7 @@ std::unique_ptr<AudioOutputDelegate> CreateFakeDelegate( TEST(MojoAudioOutputStreamProviderTest, AcquireTwice_BadMessage) { base::test::SingleThreadTaskEnvironment task_environment; bool got_bad_message = false; - mojo::core::SetDefaultProcessErrorCallback( + mojo::SetDefaultProcessErrorHandler( base::BindRepeating([](bool* got_bad_message, const std::string& s) { *got_bad_message = true; }, &got_bad_message)); @@ -104,15 +105,14 @@ TEST(MojoAudioOutputStreamProviderTest, AcquireTwice_BadMessage) { EXPECT_TRUE(got_bad_message); Mock::VerifyAndClear(&deleter); - mojo::core::SetDefaultProcessErrorCallback( - mojo::core::ProcessErrorCallback()); + mojo::SetDefaultProcessErrorHandler(base::NullCallback()); } TEST(MojoAudioOutputStreamProviderTest, Bitstream_BadMessageOnNonAndoirdPlatforms) { base::test::SingleThreadTaskEnvironment task_environment; bool got_bad_message = false; - mojo::core::SetDefaultProcessErrorCallback( + mojo::SetDefaultProcessErrorHandler( base::BindRepeating([](bool* got_bad_message, const std::string& s) { *got_bad_message = true; }, &got_bad_message)); @@ -145,8 +145,7 @@ TEST(MojoAudioOutputStreamProviderTest, EXPECT_TRUE(got_bad_message); Mock::VerifyAndClear(&deleter); #endif - mojo::core::SetDefaultProcessErrorCallback( - mojo::core::ProcessErrorCallback()); + mojo::SetDefaultProcessErrorHandler(base::NullCallback()); } } // namespace media diff --git a/chromium/media/mojo/services/mojo_cdm_helper.cc b/chromium/media/mojo/services/mojo_cdm_helper.cc index 2d5761e44e5..ac7c20ea8e5 100644 --- a/chromium/media/mojo/services/mojo_cdm_helper.cc +++ b/chromium/media/mojo/services/mojo_cdm_helper.cc @@ -4,6 +4,7 @@ #include "media/mojo/services/mojo_cdm_helper.h" +#include "base/macros.h" #include "base/stl_util.h" #include "media/base/cdm_context.h" #include "media/cdm/cdm_helpers.h" @@ -37,6 +38,15 @@ cdm::FileIO* MojoCdmHelper::CreateCdmFileIO(cdm::FileIOClient* client) { return cdm_file_io; } +url::Origin MojoCdmHelper::GetCdmOrigin() { + url::Origin cdm_origin; + // Since the CDM is created asynchronously, by the time this function is + // called, the render frame host in the browser process may already be gone. + // It's safe to ignore the error since the origin is used for crash reporting. + ignore_result(frame_interfaces_->GetCdmOrigin(&cdm_origin)); + return cdm_origin; +} + cdm::Buffer* MojoCdmHelper::CreateCdmBuffer(size_t capacity) { return GetAllocator()->CreateCdmBuffer(capacity); } diff --git a/chromium/media/mojo/services/mojo_cdm_helper.h b/chromium/media/mojo/services/mojo_cdm_helper.h index 25bf7d13790..0676ec9f1e6 100644 --- a/chromium/media/mojo/services/mojo_cdm_helper.h +++ b/chromium/media/mojo/services/mojo_cdm_helper.h @@ -35,6 +35,7 @@ class MEDIA_MOJO_EXPORT MojoCdmHelper final : public CdmAuxiliaryHelper, // CdmAuxiliaryHelper implementation. void SetFileReadCB(FileReadCB file_read_cb) final; cdm::FileIO* CreateCdmFileIO(cdm::FileIOClient* client) final; + url::Origin GetCdmOrigin() final; cdm::Buffer* CreateCdmBuffer(size_t capacity) final; std::unique_ptr<VideoFrameImpl> CreateCdmVideoFrame() final; void QueryStatus(QueryStatusCB callback) final; diff --git a/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc b/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc index 1b52d254b27..b41e3160d72 100644 --- a/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc +++ b/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc @@ -15,6 +15,7 @@ #include "mojo/public/cpp/bindings/self_owned_receiver.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" +#include "url/origin.h" using Status = cdm::FileIOClient::Status; @@ -73,6 +74,7 @@ class TestFrameInterfaceFactory : public mojom::FrameInterfaceFactory { mojo::MakeSelfOwnedReceiver(std::make_unique<MockCdmStorage>(), std::move(receiver)); } + void GetCdmOrigin(GetCdmOriginCallback callback) override {} void BindEmbedderReceiver(mojo::GenericPendingReceiver) override {} }; diff --git a/chromium/media/mojo/services/mojo_cdm_service.cc b/chromium/media/mojo/services/mojo_cdm_service.cc index 16e60b82eb9..ba20eea6c0f 100644 --- a/chromium/media/mojo/services/mojo_cdm_service.cc +++ b/chromium/media/mojo/services/mojo_cdm_service.cc @@ -19,7 +19,6 @@ #include "media/mojo/common/media_type_converters.h" #include "media/mojo/services/mojo_cdm_service_context.h" #include "mojo/public/cpp/bindings/pending_remote.h" -#include "url/origin.h" namespace media { @@ -58,7 +57,6 @@ void MojoCdmService::SetClient( } void MojoCdmService::Initialize(const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, InitializeCallback callback) { DVLOG(1) << __func__ << ": " << key_system; @@ -68,7 +66,7 @@ void MojoCdmService::Initialize(const std::string& key_system, auto weak_this = weak_factory_.GetWeakPtr(); cdm_factory_->Create( - key_system, security_origin, cdm_config, + key_system, cdm_config, base::Bind(&MojoCdmService::OnSessionMessage, weak_this), base::Bind(&MojoCdmService::OnSessionClosed, weak_this), base::Bind(&MojoCdmService::OnSessionKeysChange, weak_this), diff --git a/chromium/media/mojo/services/mojo_cdm_service.h b/chromium/media/mojo/services/mojo_cdm_service.h index 20903a51b13..c38939f3966 100644 --- a/chromium/media/mojo/services/mojo_cdm_service.h +++ b/chromium/media/mojo/services/mojo_cdm_service.h @@ -47,7 +47,6 @@ class MEDIA_MOJO_EXPORT MojoCdmService : public mojom::ContentDecryptionModule { mojo::PendingAssociatedRemote<mojom::ContentDecryptionModuleClient> client) final; void Initialize(const std::string& key_system, - const url::Origin& security_origin, const CdmConfig& cdm_config, InitializeCallback callback) final; void SetServerCertificate(const std::vector<uint8_t>& certificate_data, diff --git a/chromium/media/mojo/services/mojo_media_client.h b/chromium/media/mojo/services/mojo_media_client.h index 03263ee3c63..0888eb397d1 100644 --- a/chromium/media/mojo/services/mojo_media_client.h +++ b/chromium/media/mojo/services/mojo_media_client.h @@ -42,6 +42,10 @@ using SupportedVideoDecoderConfigMap = base::flat_map<VideoDecoderImplementation, std::vector<SupportedVideoDecoderConfig>>; +// Provides a way for MediaService to create concrete (e.g. platform specific) +// media components’ implementations. When MediaService is created, a +// MojoMediaClient must be passed in so that MediaService knows how to create +// the media components. class MEDIA_MOJO_EXPORT MojoMediaClient { public: // Called before the host application is scheduled to quit. diff --git a/chromium/media/mojo/services/mojo_video_decoder_service.cc b/chromium/media/mojo/services/mojo_video_decoder_service.cc index bd3b8202578..c35d0b555c2 100644 --- a/chromium/media/mojo/services/mojo_video_decoder_service.cc +++ b/chromium/media/mojo/services/mojo_video_decoder_service.cc @@ -337,7 +337,7 @@ void MojoVideoDecoderService::OnDecoderOutput(scoped_refptr<VideoFrame> frame) { // All MojoVideoDecoder-based decoders are hardware decoders. If you're the // first to implement an out-of-process decoder that is not power efficent, // you can remove this DCHECK. - DCHECK(frame->metadata()->IsTrue(VideoFrameMetadata::POWER_EFFICIENT)); + DCHECK(frame->metadata()->power_efficient); base::Optional<base::UnguessableToken> release_token; if (frame->HasReleaseMailboxCB() && video_frame_handle_releaser_) { diff --git a/chromium/media/mojo/services/mojo_video_decoder_service.h b/chromium/media/mojo/services/mojo_video_decoder_service.h index 871b9aea43a..1285874f083 100644 --- a/chromium/media/mojo/services/mojo_video_decoder_service.h +++ b/chromium/media/mojo/services/mojo_video_decoder_service.h @@ -26,7 +26,6 @@ namespace media { -class CdmContextRef; class DecoderBuffer; class MojoCdmServiceContext; class MojoDecoderBufferReader; diff --git a/chromium/media/mojo/services/playback_events_recorder.cc b/chromium/media/mojo/services/playback_events_recorder.cc new file mode 100644 index 00000000000..6080dca225a --- /dev/null +++ b/chromium/media/mojo/services/playback_events_recorder.cc @@ -0,0 +1,142 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/mojo/services/playback_events_recorder.h" + +#include "base/metrics/user_metrics.h" +#include "base/strings/strcat.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/stringprintf.h" +#include "mojo/public/cpp/bindings/self_owned_receiver.h" + +namespace media { + +namespace { + +void RecordEventWithValueAt(const char* name, + int64_t value, + base::TimeTicks time) { + base::RecordComputedActionAt(base::StrCat({"WebEngine.Media.", name, ":", + base::NumberToString(value)}), + time); +} + +void RecordEventWithValue(const char* name, int64_t value) { + RecordEventWithValueAt(name, value, base::TimeTicks::Now()); +} + +constexpr base::TimeDelta kBitrateReportPeriod = + base::TimeDelta::FromSeconds(5); + +} // namespace + +PlaybackEventsRecorder::BitrateEstimator::BitrateEstimator() {} +PlaybackEventsRecorder::BitrateEstimator::~BitrateEstimator() {} + +void PlaybackEventsRecorder::BitrateEstimator::Update( + const PipelineStatistics& stats) { + base::TimeTicks now = base::TimeTicks::Now(); + + // The code below trusts that |stats| are valid even though they came from an + // untrusted process. That's accepable because the stats are used only to + // record metrics. + if (last_stats_) { + time_elapsed_ += now - last_stats_time_; + audio_bytes_ += + stats.audio_bytes_decoded - last_stats_->audio_bytes_decoded; + video_bytes_ += + stats.video_bytes_decoded - last_stats_->video_bytes_decoded; + if (time_elapsed_ >= kBitrateReportPeriod) { + size_t audio_bitrate_kbps = + 8 * audio_bytes_ / time_elapsed_.InMilliseconds(); + RecordEventWithValueAt("AudioBitrate", audio_bitrate_kbps, now); + + size_t video_bitrate_kbps = + 8 * video_bytes_ / time_elapsed_.InMilliseconds(); + RecordEventWithValueAt("VideoBitrate", video_bitrate_kbps, now); + + time_elapsed_ = base::TimeDelta(); + audio_bytes_ = 0; + video_bytes_ = 0; + } + } + + last_stats_ = stats; + last_stats_time_ = now; +} + +void PlaybackEventsRecorder::BitrateEstimator::OnPause() { + last_stats_ = {}; +} + +// static +void PlaybackEventsRecorder::Create( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) { + mojo::MakeSelfOwnedReceiver(std::make_unique<PlaybackEventsRecorder>(), + std::move(receiver)); +} + +PlaybackEventsRecorder::PlaybackEventsRecorder() = default; +PlaybackEventsRecorder::~PlaybackEventsRecorder() = default; + +void PlaybackEventsRecorder::OnPlaying() { + base::RecordComputedAction("WebEngine.Media.Playing"); +} + +void PlaybackEventsRecorder::OnPaused() { + base::RecordComputedAction("WebEngine.Media.Pause"); + bitrate_estimator_.OnPause(); +} + +void PlaybackEventsRecorder::OnSeeking() { + buffering_state_ = BufferingState::kInitialBuffering; +} + +void PlaybackEventsRecorder::OnEnded() { + base::RecordComputedAction("WebEngine.Media.Ended"); +} + +void PlaybackEventsRecorder::OnBuffering() { + DCHECK(buffering_state_ == BufferingState::kBuffered); + + buffering_start_time_ = base::TimeTicks::Now(); + buffering_state_ = BufferingState::kBuffering; + + bitrate_estimator_.OnPause(); +} + +void PlaybackEventsRecorder::OnBufferingComplete() { + auto now = base::TimeTicks::Now(); + + if (buffering_state_ == BufferingState::kBuffering) { + base::TimeDelta time_between_buffering = + buffering_start_time_ - last_buffering_end_time_; + RecordEventWithValueAt("PlayTimeBeforeAutoPause", + time_between_buffering.InMilliseconds(), now); + + base::TimeDelta buffering_user_time = now - buffering_start_time_; + RecordEventWithValueAt("AutoPauseTime", + buffering_user_time.InMilliseconds(), now); + } + + buffering_state_ = BufferingState::kBuffered; + last_buffering_end_time_ = now; +} + +void PlaybackEventsRecorder::OnError(PipelineStatus status) { + RecordEventWithValue("Error", status); +} + +void PlaybackEventsRecorder::OnNaturalSizeChanged(const gfx::Size& size) { + int encoded_video_resolution = (size.width() << 16) | size.height(); + base::RecordComputedAction(base::StringPrintf( + "WebEngine.Media.VideoResolution:%d", encoded_video_resolution)); +} + +void PlaybackEventsRecorder::OnPipelineStatistics( + const PipelineStatistics& stats) { + bitrate_estimator_.Update(stats); +} + +} // namespace media diff --git a/chromium/media/mojo/services/playback_events_recorder.h b/chromium/media/mojo/services/playback_events_recorder.h new file mode 100644 index 00000000000..25b5e7a3c51 --- /dev/null +++ b/chromium/media/mojo/services/playback_events_recorder.h @@ -0,0 +1,71 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_MOJO_SERVICES_PLAYBACK_EVENTS_RECORDER_H_ +#define MEDIA_MOJO_SERVICES_PLAYBACK_EVENTS_RECORDER_H_ + +#include "media/mojo/mojom/playback_events_recorder.mojom.h" +#include "media/mojo/services/media_mojo_export.h" +#include "mojo/public/cpp/bindings/pending_remote.h" +#include "mojo/public/cpp/bindings/remote.h" + +namespace media { + +class MEDIA_MOJO_EXPORT PlaybackEventsRecorder : public mojom::PlaybackEventsRecorder { + public: + static void Create( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver); + + PlaybackEventsRecorder(); + ~PlaybackEventsRecorder() final; + + PlaybackEventsRecorder(const PlaybackEventsRecorder&) = delete; + PlaybackEventsRecorder& operator=(const PlaybackEventsRecorder&) = + delete; + + // mojom::PlaybackEventsRecorder implementation. + void OnPlaying() final; + void OnPaused() final; + void OnSeeking() final; + void OnEnded() final; + void OnBuffering() final; + void OnBufferingComplete() final; + void OnError(PipelineStatus status) final; + void OnNaturalSizeChanged(const gfx::Size& size) final; + void OnPipelineStatistics(const PipelineStatistics& stats) final; + + private: + class BitrateEstimator { + public: + BitrateEstimator(); + ~BitrateEstimator(); + + void Update(const PipelineStatistics& stats); + void OnPause(); + + private: + base::TimeDelta time_elapsed_; + size_t audio_bytes_ = 0; + size_t video_bytes_ = 0; + + base::Optional<PipelineStatistics> last_stats_; + base::TimeTicks last_stats_time_; + }; + + enum class BufferingState { + kInitialBuffering, + kBuffering, + kBuffered, + }; + + BufferingState buffering_state_ = BufferingState::kInitialBuffering; + base::TimeTicks buffering_start_time_; + base::TimeTicks last_buffering_end_time_; + + BitrateEstimator bitrate_estimator_; +}; + +} // namespace media + +#endif // MEDIA_MOJO_SERVICES_PLAYBACK_EVENTS_RECORDER_H_ diff --git a/chromium/media/mojo/services/playback_events_recorder_test.cc b/chromium/media/mojo/services/playback_events_recorder_test.cc new file mode 100644 index 00000000000..5d41dc2c48c --- /dev/null +++ b/chromium/media/mojo/services/playback_events_recorder_test.cc @@ -0,0 +1,203 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/mojo/services/playback_events_recorder.h" + +#include "base/metrics/user_metrics.h" +#include "base/test/simple_test_tick_clock.h" +#include "base/test/task_environment.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +constexpr base::TimeDelta kSecond = base::TimeDelta::FromSeconds(1); + +class PlaybackEventsRecorderTest : public testing::Test { + public: + PlaybackEventsRecorderTest() + : task_environment_(base::test::TaskEnvironment::MainThreadType::IO, + base::test::TaskEnvironment::TimeSource::MOCK_TIME) { + time_base_ = base::TimeTicks::Now(); + + base::SetRecordActionTaskRunner( + task_environment_.GetMainThreadTaskRunner()); + action_callback_ = base::BindRepeating( + &PlaybackEventsRecorderTest::OnAction, base::Unretained(this)); + base::AddActionCallback(action_callback_); + } + + ~PlaybackEventsRecorderTest() override { + base::RemoveActionCallback(action_callback_); + } + + protected: + struct Event { + base::TimeTicks time; + std::string name; + + bool operator==(const Event& other) const { + return time == other.time && name == other.name; + } + }; + + void OnAction(const std::string& name, base::TimeTicks time) { + recorded_events_.push_back({time, name}); + } + + void ExpectEvents(const std::vector<Event>& expected) { + EXPECT_EQ(recorded_events_.size(), expected.size()); + size_t end = std::min(recorded_events_.size(), expected.size()); + for (size_t i = 0; i < end; ++i) { + SCOPED_TRACE(i); + EXPECT_EQ(recorded_events_[i].time, expected[i].time); + EXPECT_EQ(recorded_events_[i].name, expected[i].name); + } + } + + base::test::TaskEnvironment task_environment_; + + base::SimpleTestTickClock test_clock_; + base::TimeTicks time_base_; + + base::ActionCallback action_callback_; + PlaybackEventsRecorder recorder_; + std::vector<Event> recorded_events_; +}; + +TEST_F(PlaybackEventsRecorderTest, PlayPause) { + recorder_.OnNaturalSizeChanged(gfx::Size(640, 480)); + recorder_.OnPlaying(); + task_environment_.AdvanceClock(2 * kSecond); + recorder_.OnPaused(); + + ExpectEvents({ + // VideoResolution value should be encoded as (640 << 16) + 480. + {time_base_, "WebEngine.Media.VideoResolution:41943520"}, + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 2 * kSecond, "WebEngine.Media.Pause"}, + }); +} + +TEST_F(PlaybackEventsRecorderTest, Error) { + recorder_.OnPlaying(); + task_environment_.AdvanceClock(2 * kSecond); + recorder_.OnError(PIPELINE_ERROR_DECODE); + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 2 * kSecond, "WebEngine.Media.Error:3"}, + }); +} + +TEST_F(PlaybackEventsRecorderTest, Buffering) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + task_environment_.AdvanceClock(2 * kSecond); + recorder_.OnBuffering(); + task_environment_.AdvanceClock(3 * kSecond); + recorder_.OnBufferingComplete(); + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 5 * kSecond, + "WebEngine.Media.PlayTimeBeforeAutoPause:2000"}, + {time_base_ + 5 * kSecond, "WebEngine.Media.AutoPauseTime:3000"}, + }); +} + +TEST_F(PlaybackEventsRecorderTest, Bitrate) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + + PipelineStatistics stats; + recorder_.OnPipelineStatistics(stats); + + for (int i = 0; i < 5; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 5 * kSecond, "WebEngine.Media.AudioBitrate:40"}, + {time_base_ + 5 * kSecond, "WebEngine.Media.VideoBitrate:80"}, + }); +} + +TEST_F(PlaybackEventsRecorderTest, BitrateAfterPause) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + + PipelineStatistics stats; + recorder_.OnPipelineStatistics(stats); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + recorder_.OnPaused(); + task_environment_.AdvanceClock(10 * kSecond); + recorder_.OnPlaying(); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 3 * kSecond, "WebEngine.Media.Pause"}, + {time_base_ + 13 * kSecond, "WebEngine.Media.Playing"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.AudioBitrate:40"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.VideoBitrate:80"}, + }); +} + +TEST_F(PlaybackEventsRecorderTest, BitrateAfterBuffering) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + + PipelineStatistics stats; + recorder_.OnPipelineStatistics(stats); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + recorder_.OnBuffering(); + task_environment_.AdvanceClock(10 * kSecond); + recorder_.OnBufferingComplete(); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 13 * kSecond, + "WebEngine.Media.PlayTimeBeforeAutoPause:3000"}, + {time_base_ + 13 * kSecond, "WebEngine.Media.AutoPauseTime:10000"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.AudioBitrate:40"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.VideoBitrate:80"}, + }); +} +} // namespace media |