diff options
Diffstat (limited to 'chromium/media/capture')
50 files changed, 993 insertions, 296 deletions
diff --git a/chromium/media/capture/BUILD.gn b/chromium/media/capture/BUILD.gn index 06e45ca2886..f3b9f26f051 100644 --- a/chromium/media/capture/BUILD.gn +++ b/chromium/media/capture/BUILD.gn @@ -269,6 +269,7 @@ jumbo_component("capture_lib") { "video/chromeos/camera_hal_dispatcher_impl.h", "video/chromeos/camera_metadata_utils.cc", "video/chromeos/camera_metadata_utils.h", + "video/chromeos/capture_metadata_dispatcher.h", "video/chromeos/display_rotation_observer.cc", "video/chromeos/display_rotation_observer.h", "video/chromeos/gpu_memory_buffer_tracker.cc", diff --git a/chromium/media/capture/content/android/thread_safe_capture_oracle.cc b/chromium/media/capture/content/android/thread_safe_capture_oracle.cc index 27a1515b9e7..498f65f10d8 100644 --- a/chromium/media/capture/content/android/thread_safe_capture_oracle.cc +++ b/chromium/media/capture/content/android/thread_safe_capture_oracle.cc @@ -221,16 +221,11 @@ void ThreadSafeCaptureOracle::DidCaptureFrame( if (!should_deliver_frame || !client_) return; - frame->metadata()->SetDouble(VideoFrameMetadata::FRAME_RATE, - params_.requested_format.frame_rate); - frame->metadata()->SetTimeTicks(VideoFrameMetadata::CAPTURE_BEGIN_TIME, - capture->begin_time); - frame->metadata()->SetTimeTicks(VideoFrameMetadata::CAPTURE_END_TIME, - base::TimeTicks::Now()); - frame->metadata()->SetTimeDelta(VideoFrameMetadata::FRAME_DURATION, - capture->frame_duration); - frame->metadata()->SetTimeTicks(VideoFrameMetadata::REFERENCE_TIME, - reference_time); + frame->metadata()->frame_rate = params_.requested_format.frame_rate; + frame->metadata()->capture_begin_time = capture->begin_time; + frame->metadata()->capture_end_time = base::TimeTicks::Now(); + frame->metadata()->frame_duration = capture->frame_duration; + frame->metadata()->reference_time = reference_time; media::VideoCaptureFormat format(frame->coded_size(), params_.requested_format.frame_rate, diff --git a/chromium/media/capture/content/video_capture_oracle.cc b/chromium/media/capture/content/video_capture_oracle.cc index 2a1e3c12801..e92d28ec940 100644 --- a/chromium/media/capture/content/video_capture_oracle.cc +++ b/chromium/media/capture/content/video_capture_oracle.cc @@ -9,6 +9,7 @@ #include "base/callback.h" #include "base/compiler_specific.h" #include "base/format_macros.h" +#include "base/logging.h" #include "base/numerics/safe_conversions.h" #include "base/strings/stringprintf.h" diff --git a/chromium/media/capture/mojom/BUILD.gn b/chromium/media/capture/mojom/BUILD.gn index a2aaecee8a9..2eddaec9ea8 100644 --- a/chromium/media/capture/mojom/BUILD.gn +++ b/chromium/media/capture/mojom/BUILD.gn @@ -13,6 +13,7 @@ mojom("video_capture") { public_deps = [ "//gpu/ipc/common:interfaces", + "//media//mojo/mojom:mojom", "//mojo/public/mojom/base", "//ui/gfx/geometry/mojom", "//ui/gfx/mojom", diff --git a/chromium/media/capture/mojom/video_capture_types.mojom b/chromium/media/capture/mojom/video_capture_types.mojom index eaeff60009f..ce0943d046c 100644 --- a/chromium/media/capture/mojom/video_capture_types.mojom +++ b/chromium/media/capture/mojom/video_capture_types.mojom @@ -5,9 +5,9 @@ module media.mojom; import "gpu/ipc/common/mailbox_holder.mojom"; +import "media/mojo/mojom/media_types.mojom"; import "mojo/public/mojom/base/shared_memory.mojom"; import "mojo/public/mojom/base/time.mojom"; -import "mojo/public/mojom/base/values.mojom"; import "ui/gfx/geometry/mojom/geometry.mojom"; import "ui/gfx/mojom/buffer_types.mojom"; import "ui/gfx/mojom/color_space.mojom"; @@ -279,7 +279,7 @@ struct PlaneStrides { struct VideoFrameInfo{ mojo_base.mojom.TimeDelta timestamp; - mojo_base.mojom.DictionaryValue metadata; + VideoFrameMetadata metadata; VideoCapturePixelFormat pixel_format; gfx.mojom.Size coded_size; gfx.mojom.Rect visible_rect; @@ -293,6 +293,10 @@ struct VideoFrameInfo{ PlaneStrides? strides; }; +// Represents information about a capture device. +// |device_id| represents a unique id of a physical device. Since the same +// physical device may be accessible through different APIs |capture_api| +// disambiguates the API. struct VideoCaptureDeviceDescriptor { string display_name; string device_id; @@ -300,8 +304,12 @@ struct VideoCaptureDeviceDescriptor { VideoFacingMode facing_mode; VideoCaptureApi capture_api; VideoCaptureTransportType transport_type; + bool has_pan_tilt_zoom_supported; + bool pan_tilt_zoom_supported; }; +// Bundles a VideoCaptureDeviceDescriptor with corresponding supported +// video formats. struct VideoCaptureDeviceInfo { VideoCaptureDeviceDescriptor descriptor; array<VideoCaptureFormat> supported_formats; diff --git a/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc b/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc index a910706f1ac..d2f604851f8 100644 --- a/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc +++ b/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc @@ -1687,6 +1687,8 @@ bool StructTraits<media::mojom::VideoCaptureDeviceDescriptorDataView, return false; if (!data.ReadTransportType(&(output->transport_type))) return false; + if (data.has_pan_tilt_zoom_supported()) + output->set_pan_tilt_zoom_supported(data.pan_tilt_zoom_supported()); return true; } diff --git a/chromium/media/capture/mojom/video_capture_types_mojom_traits.h b/chromium/media/capture/mojom/video_capture_types_mojom_traits.h index ebe98f6280b..548149da828 100644 --- a/chromium/media/capture/mojom/video_capture_types_mojom_traits.h +++ b/chromium/media/capture/mojom/video_capture_types_mojom_traits.h @@ -189,6 +189,16 @@ struct COMPONENT_EXPORT(MEDIA_CAPTURE_MOJOM_TRAITS) return input.transport_type; } + static bool has_pan_tilt_zoom_supported( + const media::VideoCaptureDeviceDescriptor& input) { + return input.pan_tilt_zoom_supported().has_value(); + } + + static bool pan_tilt_zoom_supported( + const media::VideoCaptureDeviceDescriptor& input) { + return input.pan_tilt_zoom_supported().value_or(false); + } + static bool Read(media::mojom::VideoCaptureDeviceDescriptorDataView data, media::VideoCaptureDeviceDescriptor* output); }; diff --git a/chromium/media/capture/run_all_unittests.cc b/chromium/media/capture/run_all_unittests.cc index 70c36d5c855..3302a184ac3 100644 --- a/chromium/media/capture/run_all_unittests.cc +++ b/chromium/media/capture/run_all_unittests.cc @@ -5,6 +5,7 @@ #include <stdio.h> #include "base/bind.h" +#include "base/logging.h" #include "base/message_loop/message_pump_type.h" #include "base/test/launcher/unit_test_launcher.h" #include "base/test/test_suite.h" diff --git a/chromium/media/capture/video/android/video_capture_device_android.cc b/chromium/media/capture/video/android/video_capture_device_android.cc index b8897bf7a6f..e9a555e7da0 100644 --- a/chromium/media/capture/video/android/video_capture_device_android.cc +++ b/chromium/media/capture/video/android/video_capture_device_android.cc @@ -14,6 +14,7 @@ #include "base/numerics/safe_conversions.h" #include "base/strings/string_number_conversions.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/trace_event/trace_event.h" #include "media/capture/mojom/image_capture_types.h" #include "media/capture/video/android/capture_jni_headers/VideoCapture_jni.h" #include "media/capture/video/android/photo_capabilities.h" diff --git a/chromium/media/capture/video/android/video_capture_device_factory_android.cc b/chromium/media/capture/video/android/video_capture_device_factory_android.cc index be9228c9c20..8e5cba8978e 100644 --- a/chromium/media/capture/video/android/video_capture_device_factory_android.cc +++ b/chromium/media/capture/video/android/video_capture_device_factory_android.cc @@ -83,7 +83,8 @@ void VideoCaptureDeviceFactoryAndroid::GetDeviceDescriptors( display_name, device_id, "" /*model_id*/, static_cast<VideoCaptureApi>(capture_api_type), VideoCaptureTransportType::OTHER_TRANSPORT, - static_cast<VideoFacingMode>(facing_mode)); + static_cast<VideoFacingMode>(facing_mode), + /*pan_tilt_zoom_supported=*/false); // We put user-facing devices to the front of the list in order to make // them by-default preferred over environment-facing ones when no other diff --git a/chromium/media/capture/video/chromeos/camera_buffer_factory.cc b/chromium/media/capture/video/chromeos/camera_buffer_factory.cc index 5e3eceb98a7..5e497c1cc19 100644 --- a/chromium/media/capture/video/chromeos/camera_buffer_factory.cc +++ b/chromium/media/capture/video/chromeos/camera_buffer_factory.cc @@ -4,6 +4,7 @@ #include "media/capture/video/chromeos/camera_buffer_factory.h" +#include "base/stl_util.h" #include "media/capture/video/chromeos/video_capture_device_factory_chromeos.h" namespace media { @@ -14,24 +15,27 @@ CameraBufferFactory::~CameraBufferFactory() = default; std::unique_ptr<gfx::GpuMemoryBuffer> CameraBufferFactory::CreateGpuMemoryBuffer(const gfx::Size& size, - gfx::BufferFormat format) { + gfx::BufferFormat format, + gfx::BufferUsage usage) { gpu::GpuMemoryBufferManager* buf_manager = VideoCaptureDeviceFactoryChromeOS::GetBufferManager(); if (!buf_manager) { LOG(ERROR) << "GpuMemoryBufferManager not set"; return std::unique_ptr<gfx::GpuMemoryBuffer>(); } - return buf_manager->CreateGpuMemoryBuffer( - size, format, GetBufferUsage(format), gpu::kNullSurfaceHandle); + return buf_manager->CreateGpuMemoryBuffer(size, format, usage, + gpu::kNullSurfaceHandle); } // There's no good way to resolve the HAL pixel format to the platform-specific // DRM format, other than to actually allocate the buffer and see if the // allocation succeeds. ChromiumPixelFormat CameraBufferFactory::ResolveStreamBufferFormat( - cros::mojom::HalPixelFormat hal_format) { - if (resolved_hal_formats_.find(hal_format) != resolved_hal_formats_.end()) { - return resolved_hal_formats_[hal_format]; + cros::mojom::HalPixelFormat hal_format, + gfx::BufferUsage usage) { + const auto key = std::make_pair(hal_format, usage); + if (base::Contains(resolved_format_usages_, key)) { + return resolved_format_usages_[key]; } ChromiumPixelFormat kUnsupportedFormat{PIXEL_FORMAT_UNKNOWN, @@ -44,25 +48,13 @@ ChromiumPixelFormat CameraBufferFactory::ResolveStreamBufferFormat( } for (const auto& f : cr_formats) { auto buffer = CreateGpuMemoryBuffer( - gfx::Size(kDummyBufferWidth, kDummyBufferHeight), f.gfx_format); + gfx::Size(kDummyBufferWidth, kDummyBufferHeight), f.gfx_format, usage); if (buffer) { - resolved_hal_formats_[hal_format] = f; + resolved_format_usages_[key] = f; return f; } } return kUnsupportedFormat; } -// static -gfx::BufferUsage CameraBufferFactory::GetBufferUsage(gfx::BufferFormat format) { - switch (format) { - case gfx::BufferFormat::R_8: - // Usage for JPEG capture buffer backed by R8 pixel buffer. - return gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE; - default: - // Default usage for YUV camera buffer. - return gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE; - } -} - } // namespace media diff --git a/chromium/media/capture/video/chromeos/camera_buffer_factory.h b/chromium/media/capture/video/chromeos/camera_buffer_factory.h index ded3d310755..a2c2a2cb015 100644 --- a/chromium/media/capture/video/chromeos/camera_buffer_factory.h +++ b/chromium/media/capture/video/chromeos/camera_buffer_factory.h @@ -5,8 +5,8 @@ #ifndef MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_BUFFER_FACTORY_H_ #define MEDIA_CAPTURE_VIDEO_CHROMEOS_CAMERA_BUFFER_FACTORY_H_ +#include <map> #include <memory> -#include <unordered_map> #include "media/capture/video/chromeos/mojom/camera3.mojom.h" #include "media/capture/video/chromeos/pixel_format_utils.h" @@ -24,16 +24,17 @@ class CAPTURE_EXPORT CameraBufferFactory { virtual std::unique_ptr<gfx::GpuMemoryBuffer> CreateGpuMemoryBuffer( const gfx::Size& size, - gfx::BufferFormat format); + gfx::BufferFormat format, + gfx::BufferUsage usage); virtual ChromiumPixelFormat ResolveStreamBufferFormat( - cros::mojom::HalPixelFormat hal_format); - - static gfx::BufferUsage GetBufferUsage(gfx::BufferFormat format); + cros::mojom::HalPixelFormat hal_format, + gfx::BufferUsage usage); private: - std::unordered_map<cros::mojom::HalPixelFormat, ChromiumPixelFormat> - resolved_hal_formats_; + std::map<std::pair<cros::mojom::HalPixelFormat, gfx::BufferUsage>, + ChromiumPixelFormat> + resolved_format_usages_; }; } // namespace media diff --git a/chromium/media/capture/video/chromeos/camera_device_context.cc b/chromium/media/capture/video/chromeos/camera_device_context.cc index 03e98623ed5..d3312f60a47 100644 --- a/chromium/media/capture/video/chromeos/camera_device_context.cc +++ b/chromium/media/capture/video/chromeos/camera_device_context.cc @@ -50,8 +50,8 @@ void CameraDeviceContext::SubmitCapturedVideoCaptureBuffer( base::TimeDelta timestamp) { VideoFrameMetadata metadata; // All frames are pre-rotated to the display orientation. - metadata.SetRotation(VideoFrameMetadata::Key::ROTATION, - VideoRotation::VIDEO_ROTATION_0); + metadata.rotation = VideoRotation::VIDEO_ROTATION_0; + // TODO: Figure out the right color space for the camera frame. We may need // to populate the camera metadata with the color space reported by the V4L2 // device. diff --git a/chromium/media/capture/video/chromeos/camera_device_delegate.cc b/chromium/media/capture/video/chromeos/camera_device_delegate.cc index c361ae56ad9..cafc48e6c2e 100644 --- a/chromium/media/capture/video/chromeos/camera_device_delegate.cc +++ b/chromium/media/capture/video/chromeos/camera_device_delegate.cc @@ -32,6 +32,21 @@ namespace media { namespace { +constexpr char kBrightness[] = "com.google.control.brightness"; +constexpr char kBrightnessRange[] = "com.google.control.brightnessRange"; +constexpr char kContrast[] = "com.google.control.contrast"; +constexpr char kContrastRange[] = "com.google.control.contrastRange"; +constexpr char kPan[] = "com.google.control.pan"; +constexpr char kPanRange[] = "com.google.control.panRange"; +constexpr char kSaturation[] = "com.google.control.saturation"; +constexpr char kSaturationRange[] = "com.google.control.saturationRange"; +constexpr char kSharpness[] = "com.google.control.sharpness"; +constexpr char kSharpnessRange[] = "com.google.control.sharpnessRange"; +constexpr char kTilt[] = "com.google.control.tilt"; +constexpr char kTiltRange[] = "com.google.control.tiltRange"; +constexpr char kZoom[] = "com.google.control.zoom"; +constexpr char kZoomRange[] = "com.google.control.zoomRange"; + std::pair<int32_t, int32_t> GetTargetFrameRateRange( const cros::mojom::CameraMetadataPtr& static_metadata, int target_frame_rate, @@ -216,6 +231,9 @@ class CameraDeviceDelegate::StreamCaptureInterfaceImpl final const base::WeakPtr<CameraDeviceDelegate> camera_device_delegate_; }; +ResultMetadata::ResultMetadata() = default; +ResultMetadata::~ResultMetadata() = default; + CameraDeviceDelegate::CameraDeviceDelegate( VideoCaptureDeviceDescriptor device_descriptor, scoped_refptr<CameraHalDelegate> camera_hal_delegate, @@ -233,6 +251,7 @@ void CameraDeviceDelegate::AllocateAndStart( CameraDeviceContext* device_context) { DCHECK(ipc_task_runner_->BelongsToCurrentThread()); + got_result_metadata_ = false; chrome_capture_params_ = params; device_context_ = device_context; device_context_->SetState(CameraDeviceContext::State::kStarting); @@ -259,6 +278,10 @@ void CameraDeviceDelegate::AllocateAndStart( FROM_HERE, "Camera is missing required sensor orientation info"); return; } + auto rect = GetMetadataEntryAsSpan<int32_t>( + static_metadata_, + cros::mojom::CameraMetadataTag::ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE); + active_array_size_ = gfx::Rect(rect[0], rect[1], rect[2], rect[3]); device_context_->SetSensorOrientation(sensor_orientation[0]); // |device_ops_| is bound after the BindNewPipeAndPassReceiver call. @@ -324,38 +347,13 @@ void CameraDeviceDelegate::GetPhotoState( VideoCaptureDevice::GetPhotoStateCallback callback) { DCHECK(ipc_task_runner_->BelongsToCurrentThread()); - auto photo_state = mojo::CreateEmptyPhotoState(); - - if (!device_context_ || - (device_context_->GetState() != - CameraDeviceContext::State::kStreamConfigured && - device_context_->GetState() != CameraDeviceContext::State::kCapturing)) { - std::move(callback).Run(std::move(photo_state)); - return; - } - - std::vector<gfx::Size> blob_resolutions; - GetStreamResolutions( - static_metadata_, cros::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT, - cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_BLOB, &blob_resolutions); - if (blob_resolutions.empty()) { - std::move(callback).Run(std::move(photo_state)); + if (!got_result_metadata_) { + get_photo_state_queue_.push_back( + base::BindOnce(&CameraDeviceDelegate::DoGetPhotoState, + weak_ptr_factory_.GetWeakPtr(), std::move(callback))); return; } - - // Sets the correct range of min/max resolution in order to bypass checks that - // the resolution caller request should fall within the range when taking - // photos. And since we are not actually use the mechanism to get other - // resolutions, we set the step to 0.0 here. - photo_state->width->current = current_blob_resolution_.width(); - photo_state->width->min = blob_resolutions.front().width(); - photo_state->width->max = blob_resolutions.back().width(); - photo_state->width->step = 0.0; - photo_state->height->current = current_blob_resolution_.height(); - photo_state->height->min = blob_resolutions.front().height(); - photo_state->height->max = blob_resolutions.back().height(); - photo_state->height->step = 0.0; - std::move(callback).Run(std::move(photo_state)); + DoGetPhotoState(std::move(callback)); } // On success, invokes |callback| with value |true|. On failure, drops @@ -375,6 +373,55 @@ void CameraDeviceDelegate::SetPhotoOptions( return; } + auto set_vendor_int = [&](const std::string& name, bool has_field, + double value) { + if (!has_field) { + return; + } + const VendorTagInfo* info = + camera_hal_delegate_->GetVendorTagInfoByName(name); + if (info == nullptr) + return; + std::vector<uint8_t> temp(sizeof(int32_t)); + auto* temp_ptr = reinterpret_cast<int32_t*>(temp.data()); + *temp_ptr = value; + request_manager_->SetRepeatingCaptureMetadata(info->tag, info->type, 1, + std::move(temp)); + }; + set_vendor_int(kBrightness, settings->has_brightness, settings->brightness); + set_vendor_int(kContrast, settings->has_contrast, settings->contrast); + set_vendor_int(kPan, settings->has_pan, settings->pan); + set_vendor_int(kSaturation, settings->has_saturation, settings->saturation); + set_vendor_int(kSharpness, settings->has_sharpness, settings->sharpness); + set_vendor_int(kTilt, settings->has_tilt, settings->tilt); + if (settings->has_zoom && use_digital_zoom_) { + if (settings->zoom == 1) { + request_manager_->UnsetRepeatingCaptureMetadata( + cros::mojom::CameraMetadataTag::ANDROID_SCALER_CROP_REGION); + VLOG(1) << "zoom ratio 1"; + } else { + double zoom_factor = sqrt(settings->zoom); + int32_t crop_width = std::round(active_array_size_.width() / zoom_factor); + int32_t crop_height = + std::round(active_array_size_.height() / zoom_factor); + // crop from center + int32_t region[4] = {(active_array_size_.width() - crop_width) / 2, + (active_array_size_.height() - crop_height) / 2, + crop_width, crop_height}; + + request_manager_->SetRepeatingCaptureMetadata( + cros::mojom::CameraMetadataTag::ANDROID_SCALER_CROP_REGION, + cros::mojom::EntryType::TYPE_INT32, 4, + SerializeMetadataValueFromSpan(base::make_span(region, 4))); + + VLOG(1) << "zoom ratio:" << settings->zoom << " scaler.crop.region(" + << region[0] << "," << region[1] << "," << region[2] << "," + << region[3] << ")"; + } + } else { + set_vendor_int(kZoom, settings->has_zoom, settings->zoom); + } + bool is_resolution_specified = settings->has_width && settings->has_height; bool should_reconfigure_streams = is_resolution_specified && (current_blob_resolution_.IsEmpty() || @@ -483,6 +530,9 @@ void CameraDeviceDelegate::OnClosed(int32_t result) { device_context_->LogToClient(std::string("Failed to close device: ") + base::safe_strerror(-result)); } + if (request_manager_) { + request_manager_->RemoveResultMetadataObserver(this); + } ResetMojoInterface(); device_context_ = nullptr; current_blob_resolution_.SetSize(0, 0); @@ -548,6 +598,7 @@ void CameraDeviceDelegate::Initialize() { device_ops_->Initialize( std::move(callback_ops), base::BindOnce(&CameraDeviceDelegate::OnInitialized, GetWeakPtr())); + request_manager_->AddResultMetadataObserver(this); } void CameraDeviceDelegate::OnInitialized(int32_t result) { @@ -607,7 +658,8 @@ void CameraDeviceDelegate::ConfigureStreams( chrome_capture_params_.requested_format.frame_size.height(); preview_stream->format = cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_YCbCr_420_888; - preview_stream->usage = cros::mojom::GRALLOC_USAGE_HW_COMPOSER; + preview_stream->usage = cros::mojom::GRALLOC_USAGE_HW_COMPOSER | + cros::mojom::GRALLOC_USAGE_HW_VIDEO_ENCODER; preview_stream->data_space = 0; preview_stream->rotation = cros::mojom::Camera3StreamRotation::CAMERA3_STREAM_ROTATION_0; @@ -1030,20 +1082,168 @@ bool CameraDeviceDelegate::SetPointsOfInterest( }(); // TODO(shik): Respect to SCALER_CROP_REGION, which is unused now. - - auto active_array_size = [&]() { - auto rect = GetMetadataEntryAsSpan<int32_t>( - static_metadata_, - cros::mojom::CameraMetadataTag::ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE); - // (xmin, ymin, width, height) - return gfx::Rect(rect[0], rect[1], rect[2], rect[3]); - }(); - - x *= active_array_size.width() - 1; - y *= active_array_size.height() - 1; + x *= active_array_size_.width() - 1; + y *= active_array_size_.height() - 1; gfx::Point point = {static_cast<int>(x), static_cast<int>(y)}; camera_3a_controller_->SetPointOfInterest(point); return true; } +mojom::RangePtr CameraDeviceDelegate::GetControlRangeByVendorTagName( + const std::string& range_name, + const base::Optional<int32_t>& current) { + const VendorTagInfo* info = + camera_hal_delegate_->GetVendorTagInfoByName(range_name); + if (info == nullptr) { + return mojom::Range::New(); + } + auto static_val = + GetMetadataEntryAsSpan<int32_t>(static_metadata_, info->tag); + if (static_val.size() != 3) { + return mojom::Range::New(); + } + + if (!current) { + return mojom::Range::New(); + } + + mojom::RangePtr range = mojom::Range::New(); + + range->min = static_val[0]; + range->max = static_val[1]; + range->step = static_val[2]; + range->current = current.value(); + + return range; +} + +void CameraDeviceDelegate::OnResultMetadataAvailable( + const cros::mojom::CameraMetadataPtr& result_metadata) { + DCHECK(ipc_task_runner_->BelongsToCurrentThread()); + + auto get_vendor_int = + [&](const std::string& name, + const cros::mojom::CameraMetadataPtr& result_metadata, + base::Optional<int32_t>* returned_value) { + returned_value->reset(); + const VendorTagInfo* info = + camera_hal_delegate_->GetVendorTagInfoByName(name); + if (info == nullptr) + return; + auto val = GetMetadataEntryAsSpan<int32_t>(result_metadata, info->tag); + if (val.size() == 1) + *returned_value = val[0]; + }; + + get_vendor_int(kBrightness, result_metadata, &result_metadata_.brightness); + get_vendor_int(kContrast, result_metadata, &result_metadata_.contrast); + get_vendor_int(kPan, result_metadata, &result_metadata_.pan); + get_vendor_int(kSaturation, result_metadata, &result_metadata_.saturation); + get_vendor_int(kSharpness, result_metadata, &result_metadata_.sharpness); + get_vendor_int(kTilt, result_metadata, &result_metadata_.tilt); + get_vendor_int(kZoom, result_metadata, &result_metadata_.zoom); + + result_metadata_.scaler_crop_region.reset(); + auto rect = GetMetadataEntryAsSpan<int32_t>( + result_metadata, + cros::mojom::CameraMetadataTag::ANDROID_SCALER_CROP_REGION); + if (rect.size() == 4) { + result_metadata_.scaler_crop_region = + gfx::Rect(rect[0], rect[1], rect[2], rect[3]); + } + + if (!got_result_metadata_) { + for (auto& request : get_photo_state_queue_) + ipc_task_runner_->PostTask(FROM_HERE, std::move(request)); + get_photo_state_queue_.clear(); + got_result_metadata_ = true; + } +} + +void CameraDeviceDelegate::DoGetPhotoState( + VideoCaptureDevice::GetPhotoStateCallback callback) { + DCHECK(ipc_task_runner_->BelongsToCurrentThread()); + + auto photo_state = mojo::CreateEmptyPhotoState(); + + if (!device_context_ || + (device_context_->GetState() != + CameraDeviceContext::State::kStreamConfigured && + device_context_->GetState() != CameraDeviceContext::State::kCapturing)) { + std::move(callback).Run(std::move(photo_state)); + return; + } + + std::vector<gfx::Size> blob_resolutions; + GetStreamResolutions( + static_metadata_, cros::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT, + cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_BLOB, &blob_resolutions); + if (blob_resolutions.empty()) { + std::move(callback).Run(std::move(photo_state)); + return; + } + + // Sets the correct range of min/max resolution in order to bypass checks that + // the resolution caller request should fall within the range when taking + // photos. And since we are not actually use the mechanism to get other + // resolutions, we set the step to 0.0 here. + photo_state->width->current = current_blob_resolution_.width(); + photo_state->width->min = blob_resolutions.front().width(); + photo_state->width->max = blob_resolutions.back().width(); + photo_state->width->step = 0.0; + photo_state->height->current = current_blob_resolution_.height(); + photo_state->height->min = blob_resolutions.front().height(); + photo_state->height->max = blob_resolutions.back().height(); + photo_state->height->step = 0.0; + + photo_state->brightness = GetControlRangeByVendorTagName( + kBrightnessRange, result_metadata_.brightness); + photo_state->contrast = + GetControlRangeByVendorTagName(kContrastRange, result_metadata_.contrast); + photo_state->pan = + GetControlRangeByVendorTagName(kPanRange, result_metadata_.pan); + photo_state->saturation = GetControlRangeByVendorTagName( + kSaturationRange, result_metadata_.saturation); + photo_state->sharpness = GetControlRangeByVendorTagName( + kSharpnessRange, result_metadata_.sharpness); + photo_state->tilt = + GetControlRangeByVendorTagName(kTiltRange, result_metadata_.tilt); + + // For zoom part, we check the scaler.availableMaxDigitalZoom first, if there + // is no metadata or the value is one we use zoom vendor tag. + // + // https://w3c.github.io/mediacapture-image/#zoom + // + // scaler.availableMaxDigitalZoom: + // We use area ratio for this type zoom. + // + // Vendor tag zoom: + // It is used by UVC camera usually. + // The zoom unit is driver-specific for V4L2_CID_ZOOM_ABSOLUTE. + // https://www.kernel.org/doc/html/latest/media/uapi/v4l/ext-ctrls-camera.html + auto max_digital_zoom = GetMetadataEntryAsSpan<float>( + static_metadata_, cros::mojom::CameraMetadataTag:: + ANDROID_SCALER_AVAILABLE_MAX_DIGITAL_ZOOM); + if (max_digital_zoom.size() == 1 && max_digital_zoom[0] > 1 && + result_metadata_.scaler_crop_region) { + photo_state->zoom->min = 1; + photo_state->zoom->max = max_digital_zoom[0] * max_digital_zoom[0]; + photo_state->zoom->step = 0.1; + photo_state->zoom->current = + (active_array_size_.width() / + (float)result_metadata_.scaler_crop_region->width()) * + (active_array_size_.height() / + (float)result_metadata_.scaler_crop_region->height()); + // get 0.1 precision + photo_state->zoom->current = round(photo_state->zoom->current * 10) / 10; + use_digital_zoom_ = true; + } else { + photo_state->zoom = + GetControlRangeByVendorTagName(kZoomRange, result_metadata_.zoom); + use_digital_zoom_ = false; + } + + std::move(callback).Run(std::move(photo_state)); +} + } // namespace media diff --git a/chromium/media/capture/video/chromeos/camera_device_delegate.h b/chromium/media/capture/video/chromeos/camera_device_delegate.h index a8aceeea4c0..d261b2b96cc 100644 --- a/chromium/media/capture/video/chromeos/camera_device_delegate.h +++ b/chromium/media/capture/video/chromeos/camera_device_delegate.h @@ -11,6 +11,7 @@ #include "base/memory/weak_ptr.h" #include "base/optional.h" #include "base/single_thread_task_runner.h" +#include "media/capture/video/chromeos/capture_metadata_dispatcher.h" #include "media/capture/video/chromeos/mojom/camera3.mojom.h" #include "media/capture/video/chromeos/mojom/camera_common.mojom.h" #include "media/capture/video/video_capture_device.h" @@ -35,6 +36,22 @@ enum class StreamType : uint64_t { kUnknown, }; +// The metadata might be large so clone a whole metadata might be relatively +// expensive. We only keep the needed data by this structure. +struct ResultMetadata { + ResultMetadata(); + ~ResultMetadata(); + + base::Optional<int32_t> brightness; + base::Optional<int32_t> contrast; + base::Optional<int32_t> pan; + base::Optional<int32_t> saturation; + base::Optional<int32_t> sharpness; + base::Optional<int32_t> tilt; + base::Optional<int32_t> zoom; + base::Optional<gfx::Rect> scaler_crop_region; +}; + // Returns true if the given stream type is an input stream. bool IsInputStream(StreamType stream_type); @@ -71,7 +88,8 @@ class CAPTURE_EXPORT StreamCaptureInterface { // AllocateAndStart of VideoCaptureDeviceArcChromeOS runs on. All the methods // in CameraDeviceDelegate run on |ipc_task_runner_| and hence all the // access to member variables is sequenced. -class CAPTURE_EXPORT CameraDeviceDelegate final { +class CAPTURE_EXPORT CameraDeviceDelegate final + : public CaptureMetadataDispatcher::ResultMetadataObserver { public: CameraDeviceDelegate( VideoCaptureDeviceDescriptor device_descriptor, @@ -79,7 +97,7 @@ class CAPTURE_EXPORT CameraDeviceDelegate final { scoped_refptr<base::SingleThreadTaskRunner> ipc_task_runner, CameraAppDeviceImpl* camera_app_device); - ~CameraDeviceDelegate(); + ~CameraDeviceDelegate() final; // Delegation methods for the VideoCaptureDevice interface. void AllocateAndStart(const VideoCaptureParams& params, @@ -175,6 +193,18 @@ class CAPTURE_EXPORT CameraDeviceDelegate final { bool SetPointsOfInterest( const std::vector<mojom::Point2DPtr>& points_of_interest); + // This function gets the TYPE_INT32[3] array of [max, min, step] from static + // metadata by |range_name| and current value of |current|. + mojom::RangePtr GetControlRangeByVendorTagName( + const std::string& range_name, + const base::Optional<int32_t>& current); + + // CaptureMetadataDispatcher::ResultMetadataObserver implementation. + void OnResultMetadataAvailable( + const cros::mojom::CameraMetadataPtr& result_metadata) final; + + void DoGetPhotoState(VideoCaptureDevice::GetPhotoStateCallback callback); + const VideoCaptureDeviceDescriptor device_descriptor_; // Current configured resolution of BLOB stream. @@ -208,6 +238,13 @@ class CAPTURE_EXPORT CameraDeviceDelegate final { CameraAppDeviceImpl* camera_app_device_; // Weak. + // GetPhotoState requests waiting for |got_result_metadata_| to be served. + std::vector<base::OnceClosure> get_photo_state_queue_; + bool got_result_metadata_; + bool use_digital_zoom_; + ResultMetadata result_metadata_; + gfx::Rect active_array_size_; + base::WeakPtrFactory<CameraDeviceDelegate> weak_ptr_factory_{this}; DISALLOW_IMPLICIT_CONSTRUCTORS(CameraDeviceDelegate); diff --git a/chromium/media/capture/video/chromeos/camera_device_delegate_unittest.cc b/chromium/media/capture/video/chromeos/camera_device_delegate_unittest.cc index c0777cb92fa..3371f40ff01 100644 --- a/chromium/media/capture/video/chromeos/camera_device_delegate_unittest.cc +++ b/chromium/media/capture/video/chromeos/camera_device_delegate_unittest.cc @@ -249,6 +249,17 @@ class CameraDeviceDelegateTest : public ::testing::Test { entry->data.assign(as_int8, as_int8 + entry->count * sizeof(int32_t)); static_metadata->entries->push_back(std::move(entry)); + entry = cros::mojom::CameraMetadataEntry::New(); + entry->index = 5; + entry->tag = + cros::mojom::CameraMetadataTag::ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE; + entry->type = cros::mojom::EntryType::TYPE_INT32; + entry->count = 4; + std::vector<int32_t> active_array_size = {0, 0, 1920, 1080}; + as_int8 = reinterpret_cast<uint8_t*>(active_array_size.data()); + entry->data.assign(as_int8, as_int8 + entry->count * sizeof(int32_t)); + static_metadata->entries->push_back(std::move(entry)); + switch (camera_id) { case 0: camera_info->facing = cros::mojom::CameraFacing::CAMERA_FACING_FRONT; @@ -363,9 +374,10 @@ class CameraDeviceDelegateTest : public ::testing::Test { Invoke(this, &CameraDeviceDelegateTest::ConfigureFakeStreams)); EXPECT_CALL( mock_gpu_memory_buffer_manager_, - CreateGpuMemoryBuffer(_, gfx::BufferFormat::YUV_420_BIPLANAR, - gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, - gpu::kNullSurfaceHandle)) + CreateGpuMemoryBuffer( + _, gfx::BufferFormat::YUV_420_BIPLANAR, + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE, + gpu::kNullSurfaceHandle)) .Times(1) .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager:: CreateFakeGpuMemoryBuffer)); @@ -379,10 +391,11 @@ class CameraDeviceDelegateTest : public ::testing::Test { CreateFakeGpuMemoryBuffer)); EXPECT_CALL( mock_gpu_memory_buffer_manager_, - CreateGpuMemoryBuffer(gfx::Size(kDefaultWidth, kDefaultHeight), - gfx::BufferFormat::YUV_420_BIPLANAR, - gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, - gpu::kNullSurfaceHandle)) + CreateGpuMemoryBuffer( + gfx::Size(kDefaultWidth, kDefaultHeight), + gfx::BufferFormat::YUV_420_BIPLANAR, + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE, + gpu::kNullSurfaceHandle)) .Times(1) .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager:: CreateFakeGpuMemoryBuffer)); diff --git a/chromium/media/capture/video/chromeos/camera_hal_delegate.cc b/chromium/media/capture/video/chromeos/camera_hal_delegate.cc index adf4955a9d0..90b72382367 100644 --- a/chromium/media/capture/video/chromeos/camera_hal_delegate.cc +++ b/chromium/media/capture/video/chromeos/camera_hal_delegate.cc @@ -250,8 +250,13 @@ void CameraHalDelegate::GetSupportedFormats( } float max_fps = 1.0 * 1000000000LL / duration; + // There's no consumer information here to determine the buffer usage, so + // hard-code the usage that all the clients should be using. + constexpr gfx::BufferUsage kClientBufferUsage = + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE; const ChromiumPixelFormat cr_format = - camera_buffer_factory_->ResolveStreamBufferFormat(hal_format); + camera_buffer_factory_->ResolveStreamBufferFormat(hal_format, + kClientBufferUsage); if (cr_format.video_format == PIXEL_FORMAT_UNKNOWN) { continue; } @@ -340,6 +345,7 @@ void CameraHalDelegate::GetDeviceDescriptors( // about malformed values. } } + desc.set_pan_tilt_zoom_supported(IsPanTiltZoomSupported(camera_info)); device_id_to_camera_id_[desc.device_id] = camera_id; device_descriptors->push_back(desc); } @@ -350,6 +356,41 @@ void CameraHalDelegate::GetDeviceDescriptors( DVLOG(1) << "Number of device descriptors: " << device_descriptors->size(); } +bool CameraHalDelegate::IsPanTiltZoomSupported( + const cros::mojom::CameraInfoPtr& camera_info) { + auto is_vendor_range_valid = [&](const std::string& key) -> bool { + const VendorTagInfo* info = vendor_tag_ops_delegate_.GetInfoByName(key); + if (info == nullptr) + return false; + auto range = GetMetadataEntryAsSpan<int32_t>( + camera_info->static_camera_characteristics, info->tag); + return range.size() == 3 && range[0] < range[1]; + }; + + if (is_vendor_range_valid("com.google.control.panRange")) + return true; + + if (is_vendor_range_valid("com.google.control.tiltRange")) + return true; + + if (is_vendor_range_valid("com.google.control.zoomRange")) + return true; + + auto scaler_crop_region = GetMetadataEntryAsSpan<int32_t>( + camera_info->static_camera_characteristics, + cros::mojom::CameraMetadataTag::ANDROID_SCALER_CROP_REGION); + auto max_digital_zoom = GetMetadataEntryAsSpan<float>( + camera_info->static_camera_characteristics, + cros::mojom::CameraMetadataTag:: + ANDROID_SCALER_AVAILABLE_MAX_DIGITAL_ZOOM); + if (max_digital_zoom.size() == 1 && max_digital_zoom[0] > 1 && + scaler_crop_region.size() == 4) { + return true; + } + + return false; +} + cros::mojom::CameraInfoPtr CameraHalDelegate::GetCameraInfoFromDeviceId( const std::string& device_id) { base::AutoLock lock(camera_info_lock_); @@ -364,6 +405,11 @@ cros::mojom::CameraInfoPtr CameraHalDelegate::GetCameraInfoFromDeviceId( return it->second.Clone(); } +const VendorTagInfo* CameraHalDelegate::GetVendorTagInfoByName( + const std::string& full_name) { + return vendor_tag_ops_delegate_.GetInfoByName(full_name); +} + void CameraHalDelegate::OpenDevice( int32_t camera_id, mojo::PendingReceiver<cros::mojom::Camera3DeviceOps> device_ops_receiver, diff --git a/chromium/media/capture/video/chromeos/camera_hal_delegate.h b/chromium/media/capture/video/chromeos/camera_hal_delegate.h index bcd28a53fae..014b830f67d 100644 --- a/chromium/media/capture/video/chromeos/camera_hal_delegate.h +++ b/chromium/media/capture/video/chromeos/camera_hal_delegate.h @@ -81,10 +81,15 @@ class CAPTURE_EXPORT CameraHalDelegate final // Gets camera id from device id. Returns -1 on error. int GetCameraIdFromDeviceId(const std::string& device_id); + // Returns true if either pan, tilt, or zoom camera capability is supported. + bool IsPanTiltZoomSupported(const cros::mojom::CameraInfoPtr& camera_info); + // Gets the camera info of |device_id|. Returns null CameraInfoPtr on error. cros::mojom::CameraInfoPtr GetCameraInfoFromDeviceId( const std::string& device_id); + const VendorTagInfo* GetVendorTagInfoByName(const std::string& full_name); + private: friend class base::RefCountedThreadSafe<CameraHalDelegate>; diff --git a/chromium/media/capture/video/chromeos/camera_hal_delegate_unittest.cc b/chromium/media/capture/video/chromeos/camera_hal_delegate_unittest.cc index ab3eccfd296..07b70efb893 100644 --- a/chromium/media/capture/video/chromeos/camera_hal_delegate_unittest.cc +++ b/chromium/media/capture/video/chromeos/camera_hal_delegate_unittest.cc @@ -241,9 +241,10 @@ TEST_F(CameraHalDelegateTest, GetBuiltinCameraInfo) { // |model_id| are set properly according to the vendor tags. EXPECT_CALL(mock_gpu_memory_buffer_manager_, - CreateGpuMemoryBuffer(_, gfx::BufferFormat::YUV_420_BIPLANAR, - gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, - gpu::kNullSurfaceHandle)) + CreateGpuMemoryBuffer( + _, gfx::BufferFormat::YUV_420_BIPLANAR, + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE, + gpu::kNullSurfaceHandle)) .Times(1) .WillOnce(Invoke(&unittest_internal::MockGpuMemoryBufferManager:: CreateFakeGpuMemoryBuffer)); diff --git a/chromium/media/capture/video/chromeos/capture_metadata_dispatcher.h b/chromium/media/capture/video/chromeos/capture_metadata_dispatcher.h new file mode 100644 index 00000000000..dd17930740d --- /dev/null +++ b/chromium/media/capture/video/chromeos/capture_metadata_dispatcher.h @@ -0,0 +1,43 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// +#ifndef MEDIA_CAPTURE_VIDEO_CHROMEOS_CAPTURE_METADATA_DISPATCHER_H_ +#define MEDIA_CAPTURE_VIDEO_CHROMEOS_CAPTURE_METADATA_DISPATCHER_H_ + +#include "media/capture/capture_export.h" +#include "media/capture/video/chromeos/mojom/camera_common.mojom.h" + +namespace media { + +// Interface that provides API to let Camera3AController and +// CameraDeviceDelegate to update the metadata that will be sent with capture +// request. +class CAPTURE_EXPORT CaptureMetadataDispatcher { + public: + class ResultMetadataObserver { + public: + virtual ~ResultMetadataObserver() {} + virtual void OnResultMetadataAvailable( + const cros::mojom::CameraMetadataPtr&) = 0; + }; + + virtual ~CaptureMetadataDispatcher() {} + virtual void AddResultMetadataObserver(ResultMetadataObserver* observer) = 0; + virtual void RemoveResultMetadataObserver( + ResultMetadataObserver* observer) = 0; + virtual void SetCaptureMetadata(cros::mojom::CameraMetadataTag tag, + cros::mojom::EntryType type, + size_t count, + std::vector<uint8_t> value) = 0; + virtual void SetRepeatingCaptureMetadata(cros::mojom::CameraMetadataTag tag, + cros::mojom::EntryType type, + size_t count, + std::vector<uint8_t> value) = 0; + virtual void UnsetRepeatingCaptureMetadata( + cros::mojom::CameraMetadataTag tag) = 0; +}; + +} // namespace media + +#endif // MEDIA_CAPTURE_VIDEO_CHROMEOS_CAPTURE_METADATA_DISPATCHER_H_ diff --git a/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc b/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc index a1be18153d4..cbf8f6ef13c 100644 --- a/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc +++ b/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc @@ -25,7 +25,15 @@ bool GpuMemoryBufferTracker::Init(const gfx::Size& dimensions, << VideoPixelFormatToString(format); return false; } - buffer_ = buffer_factory_.CreateGpuMemoryBuffer(dimensions, *gfx_format); + // There's no consumer information here to determine the precise buffer usage, + // so we try the usage flag that covers all use cases. + // JPEG capture buffer is backed by R8 pixel buffer. + const gfx::BufferUsage usage = + *gfx_format == gfx::BufferFormat::R_8 + ? gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE + : gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE; + buffer_ = + buffer_factory_.CreateGpuMemoryBuffer(dimensions, *gfx_format, usage); if (!buffer_) { NOTREACHED() << "Failed to create GPU memory buffer"; return false; diff --git a/chromium/media/capture/video/chromeos/mojom/BUILD.gn b/chromium/media/capture/video/chromeos/mojom/BUILD.gn index f98d89e6e0d..cb72cea2f1b 100644 --- a/chromium/media/capture/video/chromeos/mojom/BUILD.gn +++ b/chromium/media/capture/video/chromeos/mojom/BUILD.gn @@ -17,7 +17,6 @@ mojom("cros_camera") { deps = [ "//components/chromeos_camera/common", "//media/capture/mojom:image_capture", - "//media/mojo/mojom", "//ui/gfx/geometry/mojom", "//ui/gfx/range/mojom", ] diff --git a/chromium/media/capture/video/chromeos/mojom/camera3.mojom b/chromium/media/capture/video/chromeos/mojom/camera3.mojom index 2f3fdfb4bcb..5e7e6ded75d 100644 --- a/chromium/media/capture/video/chromeos/mojom/camera3.mojom +++ b/chromium/media/capture/video/chromeos/mojom/camera3.mojom @@ -16,6 +16,7 @@ const uint32 GRALLOC_USAGE_SW_READ_OFTEN = 0x00000003; const uint32 GRALLOC_USAGE_SW_WRITE_NEVER = 0x00000000; const uint32 GRALLOC_USAGE_SW_WRITE_OFTEN = 0x00000030; const uint32 GRALLOC_USAGE_HW_COMPOSER = 0x00000800; +const uint32 GRALLOC_USAGE_HW_VIDEO_ENCODER = 0x00010000; const uint32 GRALLOC_USAGE_HW_CAMERA_WRITE = 0x00020000; const uint32 GRALLOC_USAGE_HW_CAMERA_READ = 0x00040000; // A private gralloc usage flag to force allocation of YUV420 buffer. This diff --git a/chromium/media/capture/video/chromeos/request_manager.h b/chromium/media/capture/video/chromeos/request_manager.h index 0f2c89e1601..f71699d7a0b 100644 --- a/chromium/media/capture/video/chromeos/request_manager.h +++ b/chromium/media/capture/video/chromeos/request_manager.h @@ -17,6 +17,7 @@ #include "media/capture/mojom/image_capture.mojom.h" #include "media/capture/video/chromeos/camera_app_device_impl.h" #include "media/capture/video/chromeos/camera_device_delegate.h" +#include "media/capture/video/chromeos/capture_metadata_dispatcher.h" #include "media/capture/video/chromeos/mojom/camera3.mojom.h" #include "media/capture/video/chromeos/mojom/camera_app.mojom.h" #include "media/capture/video/chromeos/request_builder.h" @@ -44,32 +45,6 @@ constexpr int32_t kMinConfiguredStreams = 1; // Maximum configured streams could contain two optional YUV streams. constexpr int32_t kMaxConfiguredStreams = 4; -// Interface that provides API to let Camera3AController to update the metadata -// that will be sent with capture request. -class CAPTURE_EXPORT CaptureMetadataDispatcher { - public: - class ResultMetadataObserver { - public: - virtual ~ResultMetadataObserver() {} - virtual void OnResultMetadataAvailable( - const cros::mojom::CameraMetadataPtr&) = 0; - }; - - virtual ~CaptureMetadataDispatcher() {} - virtual void AddResultMetadataObserver(ResultMetadataObserver* observer) = 0; - virtual void RemoveResultMetadataObserver( - ResultMetadataObserver* observer) = 0; - virtual void SetCaptureMetadata(cros::mojom::CameraMetadataTag tag, - cros::mojom::EntryType type, - size_t count, - std::vector<uint8_t> value) = 0; - virtual void SetRepeatingCaptureMetadata(cros::mojom::CameraMetadataTag tag, - cros::mojom::EntryType type, - size_t count, - std::vector<uint8_t> value) = 0; - virtual void UnsetRepeatingCaptureMetadata( - cros::mojom::CameraMetadataTag tag) = 0; -}; // RequestManager is responsible for managing the flow for sending capture // requests and receiving capture results. Having RequestBuilder to build diff --git a/chromium/media/capture/video/chromeos/request_manager_unittest.cc b/chromium/media/capture/video/chromeos/request_manager_unittest.cc index b62c2cd9ed1..74fad9dca4d 100644 --- a/chromium/media/capture/video/chromeos/request_manager_unittest.cc +++ b/chromium/media/capture/video/chromeos/request_manager_unittest.cc @@ -62,15 +62,15 @@ class FakeCameraBufferFactory : public CameraBufferFactory { } std::unique_ptr<gfx::GpuMemoryBuffer> CreateGpuMemoryBuffer( const gfx::Size& size, - gfx::BufferFormat format) override { + gfx::BufferFormat format, + gfx::BufferUsage usage) override { return unittest_internal::MockGpuMemoryBufferManager:: - CreateFakeGpuMemoryBuffer(size, format, - gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, - gpu::kNullSurfaceHandle); + CreateFakeGpuMemoryBuffer(size, format, usage, gpu::kNullSurfaceHandle); } ChromiumPixelFormat ResolveStreamBufferFormat( - cros::mojom::HalPixelFormat hal_format) override { + cros::mojom::HalPixelFormat hal_format, + gfx::BufferUsage usage) override { return ChromiumPixelFormat{PIXEL_FORMAT_NV12, gfx::BufferFormat::YUV_420_BIPLANAR}; } diff --git a/chromium/media/capture/video/chromeos/stream_buffer_manager.cc b/chromium/media/capture/video/chromeos/stream_buffer_manager.cc index c1afda42dab..184d9f57f6c 100644 --- a/chromium/media/capture/video/chromeos/stream_buffer_manager.cc +++ b/chromium/media/capture/video/chromeos/stream_buffer_manager.cc @@ -41,7 +41,9 @@ StreamBufferManager::~StreamBufferManager() { } void StreamBufferManager::ReserveBuffer(StreamType stream_type) { - if (video_capture_use_gmb_) { + // The YUV output buffer for reprocessing is not passed to client, so can be + // allocated by the local buffer factory without zero-copy concerns. + if (video_capture_use_gmb_ && stream_type != StreamType::kYUVOutput) { ReserveBufferFromPool(stream_type); } else { ReserveBufferFromFactory(stream_type); @@ -154,8 +156,8 @@ StreamBufferManager::AcquireBufferForClientById(StreamType stream_type, DCHECK(gfx_format); auto rotated_gmb = gmb_support_->CreateGpuMemoryBufferImplFromHandle( rotated_buffer.handle_provider->GetGpuMemoryBufferHandle(), - format->frame_size, *gfx_format, - CameraBufferFactory::GetBufferUsage(*gfx_format), base::NullCallback()); + format->frame_size, *gfx_format, stream_context->buffer_usage, + base::NullCallback()); if (!rotated_gmb || !rotated_gmb->Map()) { DLOG(WARNING) << "Failed to map rotated buffer"; @@ -238,19 +240,19 @@ void StreamBufferManager::SetUpStreamsAndBuffers( stream_context->capture_format = capture_format; stream_context->stream = std::move(stream); - const ChromiumPixelFormat stream_format = - camera_buffer_factory_->ResolveStreamBufferFormat( - stream_context->stream->format); - // Internally we keep track of the VideoPixelFormat that's actually - // supported by the camera instead of the one requested by the client. - stream_context->capture_format.pixel_format = stream_format.video_format; - switch (stream_type) { case StreamType::kPreviewOutput: + stream_context->buffer_dimension = gfx::Size( + stream_context->stream->width, stream_context->stream->height); + stream_context->buffer_usage = + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE; + break; case StreamType::kYUVInput: case StreamType::kYUVOutput: stream_context->buffer_dimension = gfx::Size( stream_context->stream->width, stream_context->stream->height); + stream_context->buffer_usage = + gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE; break; case StreamType::kJpegOutput: { auto jpeg_size = GetMetadataEntryAsSpan<int32_t>( @@ -258,12 +260,21 @@ void StreamBufferManager::SetUpStreamsAndBuffers( cros::mojom::CameraMetadataTag::ANDROID_JPEG_MAX_SIZE); CHECK_EQ(jpeg_size.size(), 1u); stream_context->buffer_dimension = gfx::Size(jpeg_size[0], 1); + stream_context->buffer_usage = + gfx::BufferUsage::CAMERA_AND_CPU_READ_WRITE; break; } default: { NOTREACHED(); } } + const ChromiumPixelFormat stream_format = + camera_buffer_factory_->ResolveStreamBufferFormat( + stream_context->stream->format, stream_context->buffer_usage); + // Internally we keep track of the VideoPixelFormat that's actually + // supported by the camera instead of the one requested by the client. + stream_context->capture_format.pixel_format = stream_format.video_format; + stream_context_[stream_type] = std::move(stream_context); // For input stream, there is no need to allocate buffers. @@ -381,7 +392,8 @@ void StreamBufferManager::ReserveBufferFromFactory(StreamType stream_type) { return; } auto gmb = camera_buffer_factory_->CreateGpuMemoryBuffer( - stream_context->buffer_dimension, *gfx_format); + stream_context->buffer_dimension, *gfx_format, + stream_context->buffer_usage); if (!gmb) { device_context_->SetErrorState( media::VideoCaptureError:: @@ -426,7 +438,7 @@ void StreamBufferManager::ReserveBufferFromPool(StreamType stream_type) { auto gmb = gmb_support_->CreateGpuMemoryBufferImplFromHandle( vcd_buffer.handle_provider->GetGpuMemoryBufferHandle(), stream_context->buffer_dimension, *gfx_format, - CameraBufferFactory::GetBufferUsage(*gfx_format), base::NullCallback()); + stream_context->buffer_usage, base::NullCallback()); stream_context->free_buffers.push(vcd_buffer.id); stream_context->buffers.insert(std::make_pair( vcd_buffer.id, BufferPair(std::move(gmb), std::move(vcd_buffer)))); diff --git a/chromium/media/capture/video/chromeos/stream_buffer_manager.h b/chromium/media/capture/video/chromeos/stream_buffer_manager.h index bd89f891ab5..cc468903a7a 100644 --- a/chromium/media/capture/video/chromeos/stream_buffer_manager.h +++ b/chromium/media/capture/video/chromeos/stream_buffer_manager.h @@ -131,6 +131,8 @@ class CAPTURE_EXPORT StreamBufferManager final { cros::mojom::Camera3StreamPtr stream; // The dimension of the buffer layout. gfx::Size buffer_dimension; + // The usage of the buffer. + gfx::BufferUsage buffer_usage; // The allocated buffer pairs. std::map<int, BufferPair> buffers; // The free buffers of this stream. The queue stores keys into the diff --git a/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc b/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc index 665ac4eb39e..a8056fb047f 100644 --- a/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc +++ b/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc @@ -130,11 +130,8 @@ void VideoCaptureJpegDecoderImpl::DecodeCapturedData( out_frame->BackWithOwnedSharedMemory(std::move(out_region), std::move(out_mapping)); - out_frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE, - frame_format.frame_rate); - - out_frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME, - reference_time); + out_frame->metadata()->frame_rate = frame_format.frame_rate; + out_frame->metadata()->reference_time = reference_time; media::mojom::VideoFrameInfoPtr out_frame_info = media::mojom::VideoFrameInfo::New(); @@ -142,7 +139,7 @@ void VideoCaptureJpegDecoderImpl::DecodeCapturedData( out_frame_info->pixel_format = media::PIXEL_FORMAT_I420; out_frame_info->coded_size = dimensions; out_frame_info->visible_rect = gfx::Rect(dimensions); - out_frame_info->metadata = out_frame->metadata()->GetInternalValues().Clone(); + out_frame_info->metadata = *(out_frame->metadata()); out_frame_info->color_space = out_frame->ColorSpace(); { diff --git a/chromium/media/capture/video/fake_video_capture_device.cc b/chromium/media/capture/video/fake_video_capture_device.cc index 96beb87ae21..a6a22008f17 100644 --- a/chromium/media/capture/video/fake_video_capture_device.cc +++ b/chromium/media/capture/video/fake_video_capture_device.cc @@ -610,22 +610,28 @@ void FakePhotoDevice::GetPhotoState( photo_state->focus_distance->step = kFocusDistanceStep; photo_state->pan = mojom::Range::New(); - photo_state->pan->current = fake_device_state_->pan; - photo_state->pan->max = kMaxPan; - photo_state->pan->min = kMinPan; - photo_state->pan->step = kPanStep; + if (config_.pan_tilt_zoom_supported) { + photo_state->pan->current = fake_device_state_->pan; + photo_state->pan->max = kMaxPan; + photo_state->pan->min = kMinPan; + photo_state->pan->step = kPanStep; + } photo_state->tilt = mojom::Range::New(); - photo_state->tilt->current = fake_device_state_->tilt; - photo_state->tilt->max = kMaxTilt; - photo_state->tilt->min = kMinTilt; - photo_state->tilt->step = kTiltStep; + if (config_.pan_tilt_zoom_supported) { + photo_state->tilt->current = fake_device_state_->tilt; + photo_state->tilt->max = kMaxTilt; + photo_state->tilt->min = kMinTilt; + photo_state->tilt->step = kTiltStep; + } photo_state->zoom = mojom::Range::New(); - photo_state->zoom->current = fake_device_state_->zoom; - photo_state->zoom->max = kMaxZoom; - photo_state->zoom->min = kMinZoom; - photo_state->zoom->step = kZoomStep; + if (config_.pan_tilt_zoom_supported) { + photo_state->zoom->current = fake_device_state_->zoom; + photo_state->zoom->max = kMaxZoom; + photo_state->zoom->min = kMinZoom; + photo_state->zoom->step = kZoomStep; + } photo_state->supports_torch = false; photo_state->torch = false; diff --git a/chromium/media/capture/video/fake_video_capture_device.h b/chromium/media/capture/video/fake_video_capture_device.h index 1d419292da6..057fc2c7475 100644 --- a/chromium/media/capture/video/fake_video_capture_device.h +++ b/chromium/media/capture/video/fake_video_capture_device.h @@ -156,14 +156,10 @@ class FrameDelivererFactory { }; struct FakePhotoDeviceConfig { - FakePhotoDeviceConfig() - : should_fail_get_photo_capabilities(false), - should_fail_set_photo_options(false), - should_fail_take_photo(false) {} - - bool should_fail_get_photo_capabilities; - bool should_fail_set_photo_options; - bool should_fail_take_photo; + bool pan_tilt_zoom_supported = true; + bool should_fail_get_photo_capabilities = false; + bool should_fail_set_photo_options = false; + bool should_fail_take_photo = false; }; // Implements the photo functionality of a FakeVideoCaptureDevice diff --git a/chromium/media/capture/video/fake_video_capture_device_factory.cc b/chromium/media/capture/video/fake_video_capture_device_factory.cc index 0b9c4763041..32c6f2a4ae8 100644 --- a/chromium/media/capture/video/fake_video_capture_device_factory.cc +++ b/chromium/media/capture/video/fake_video_capture_device_factory.cc @@ -209,17 +209,18 @@ void FakeVideoCaptureDeviceFactory::GetDeviceDescriptors( device_descriptors->emplace_back( base::StringPrintf("fake_device_%d", entry_index), entry.device_id, #if defined(OS_LINUX) - VideoCaptureApi::LINUX_V4L2_SINGLE_PLANE + VideoCaptureApi::LINUX_V4L2_SINGLE_PLANE, #elif defined(OS_MACOSX) - VideoCaptureApi::MACOSX_AVFOUNDATION + VideoCaptureApi::MACOSX_AVFOUNDATION, #elif defined(OS_WIN) - VideoCaptureApi::WIN_DIRECT_SHOW + VideoCaptureApi::WIN_DIRECT_SHOW, #elif defined(OS_ANDROID) - VideoCaptureApi::ANDROID_API2_LEGACY + VideoCaptureApi::ANDROID_API2_LEGACY, #elif defined(OS_FUCHSIA) - VideoCaptureApi::UNKNOWN + VideoCaptureApi::UNKNOWN, #endif - ); + VideoCaptureTransportType::OTHER_TRANSPORT, + entry.photo_device_config.pan_tilt_zoom_supported); entry_index++; } } @@ -255,6 +256,7 @@ void FakeVideoCaptureDeviceFactory::ParseFakeDevicesConfigFromOptionsString( std::vector<gfx::Size> resolutions = ArrayToVector(kDefaultResolutions); std::vector<float> frame_rates = ArrayToVector(kDefaultFrameRates); int device_count = kDefaultDeviceCount; + FakePhotoDeviceConfig photo_device_config; FakeVideoCaptureDevice::DisplayMediaType display_media_type = FakeVideoCaptureDevice::DisplayMediaType::ANY; @@ -331,6 +333,13 @@ void FakeVideoCaptureDeviceFactory::ParseFakeDevicesConfigFromOptionsString( } else if (base::EqualsCaseInsensitiveASCII(param.back(), "browser")) { display_media_type = FakeVideoCaptureDevice::DisplayMediaType::BROWSER; } + } else if (base::EqualsCaseInsensitiveASCII(param.front(), + "hardware-support")) { + photo_device_config.pan_tilt_zoom_supported = false; + if (base::EqualsCaseInsensitiveASCII(param.back(), "pan-tilt-zoom")) + photo_device_config.pan_tilt_zoom_supported = true; + else if (!base::EqualsCaseInsensitiveASCII(param.back(), "none")) + LOG(WARNING) << "Unknown hardware support " << param.back(); } } @@ -342,6 +351,7 @@ void FakeVideoCaptureDeviceFactory::ParseFakeDevicesConfigFromOptionsString( settings.device_id = base::StringPrintf(kDefaultDeviceIdMask, device_index); AppendAllCombinationsToFormatsContainer( pixel_formats, resolutions, frame_rates, &settings.supported_formats); + settings.photo_device_config = photo_device_config; settings.display_media_type = display_media_type; config->push_back(settings); } diff --git a/chromium/media/capture/video/fake_video_capture_device_unittest.cc b/chromium/media/capture/video/fake_video_capture_device_unittest.cc index 4508a593f6c..cad438c5693 100644 --- a/chromium/media/capture/video/fake_video_capture_device_unittest.cc +++ b/chromium/media/capture/video/fake_video_capture_device_unittest.cc @@ -33,6 +33,11 @@ using ::testing::Values; namespace media { +bool operator==(const FakePhotoDeviceConfig& lhs, + const FakePhotoDeviceConfig& rhs) { + return std::memcmp(&lhs, &rhs, sizeof(lhs)) == 0; +} + namespace { class ImageCaptureClient : public base::RefCounted<ImageCaptureClient> { @@ -405,6 +410,7 @@ struct CommandLineTestData { size_t expected_device_count; FakeVideoCaptureDevice::DisplayMediaType expected_display_media_type; std::vector<VideoPixelFormat> expected_pixel_formats; + FakePhotoDeviceConfig expected_photo_device_config; }; class FakeVideoCaptureDeviceFactoryTest @@ -442,6 +448,8 @@ TEST_P(FakeVideoCaptureDeviceFactoryTest, FakeVideoCaptureDeviceFactory::ParseFakeDevicesConfigFromOptionsString( GetParam().switch_value_string, &config); for (const auto& settings : config) { + EXPECT_EQ(GetParam().expected_photo_device_config, + settings.photo_device_config); EXPECT_EQ(GetParam().expected_display_media_type, settings.display_media_type); } @@ -491,7 +499,8 @@ INSTANTIATE_TEST_SUITE_P( 5, 1u, FakeVideoCaptureDevice::DisplayMediaType::ANY, - {PIXEL_FORMAT_I420}}, + {PIXEL_FORMAT_I420}, + {true, false, false, false}}, CommandLineTestData{"fps=29.97,device-count=1", 29.97f, 1u, @@ -524,6 +533,18 @@ INSTANTIATE_TEST_SUITE_P( 0u, FakeVideoCaptureDevice::DisplayMediaType::ANY, {PIXEL_FORMAT_I420}}, + CommandLineTestData{"hardware-support=none", + 20, + 1u, + FakeVideoCaptureDevice::DisplayMediaType::ANY, + {PIXEL_FORMAT_I420}, + {false}}, + CommandLineTestData{"hardware-support=pan-tilt-zoom,fps=60", + 60, + 1u, + FakeVideoCaptureDevice::DisplayMediaType::ANY, + {PIXEL_FORMAT_I420}, + {true}}, CommandLineTestData{"display-media-type=window", 20, 1u, diff --git a/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc b/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc index 32cf755b561..ecc999c707a 100644 --- a/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc +++ b/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc @@ -7,8 +7,8 @@ #include <lib/sys/cpp/component_context.h> #include "base/check_op.h" -#include "base/fuchsia/default_context.h" #include "base/fuchsia/fuchsia_logging.h" +#include "base/fuchsia/process_context.h" #include "base/location.h" #include "base/logging.h" #include "base/strings/string_number_conversions.h" @@ -186,7 +186,7 @@ void VideoCaptureDeviceFactoryFuchsia::Initialize() { DCHECK(!device_watcher_); DCHECK(devices_.empty()); - base::fuchsia::ComponentContextForCurrentProcess()->svc()->Connect( + base::ComponentContextForProcess()->svc()->Connect( device_watcher_.NewRequest()); device_watcher_.set_error_handler(fit::bind_member( diff --git a/chromium/media/capture/video/gpu_memory_buffer_utils.cc b/chromium/media/capture/video/gpu_memory_buffer_utils.cc index 62c52b725b4..6001654d2b0 100644 --- a/chromium/media/capture/video/gpu_memory_buffer_utils.cc +++ b/chromium/media/capture/video/gpu_memory_buffer_utils.cc @@ -62,7 +62,8 @@ VideoCaptureDevice::Client::ReserveResult AllocateNV12GpuMemoryBuffer( *out_gpu_memory_buffer = gmb_support->CreateGpuMemoryBufferImplFromHandle( out_capture_buffer->handle_provider->GetGpuMemoryBufferHandle(), buffer_size, kOpaqueGfxFormat, - gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, base::NullCallback()); + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE, + base::NullCallback()); return reserve_result; } diff --git a/chromium/media/capture/video/linux/fake_v4l2_impl.cc b/chromium/media/capture/video/linux/fake_v4l2_impl.cc index ef2c38c6e27..b46b8ae41f3 100644 --- a/chromium/media/capture/video/linux/fake_v4l2_impl.cc +++ b/chromium/media/capture/video/linux/fake_v4l2_impl.cc @@ -132,7 +132,24 @@ class FakeV4L2Impl::OpenedDevice { int s_ext_ctrls(v4l2_ext_controls* control) { return kSuccessReturnValue; } - int queryctrl(v4l2_queryctrl* control) { return EINVAL; } + int queryctrl(v4l2_queryctrl* control) { + switch (control->id) { + case V4L2_CID_PAN_ABSOLUTE: + case V4L2_CID_TILT_ABSOLUTE: + case V4L2_CID_ZOOM_ABSOLUTE: + if (!config_.descriptor.pan_tilt_zoom_supported().has_value() || + !config_.descriptor.pan_tilt_zoom_supported().value()) { + return EINVAL; + } + control->flags = 0; + control->minimum = 100; + control->maximum = 400; + control->step = 1; + return 0; + default: + return EINVAL; + } + } int s_fmt(v4l2_format* format) { if (format->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) diff --git a/chromium/media/capture/video/linux/video_capture_device_factory_linux.cc b/chromium/media/capture/video/linux/video_capture_device_factory_linux.cc index aab890307c1..60c8597901d 100644 --- a/chromium/media/capture/video/linux/video_capture_device_factory_linux.cc +++ b/chromium/media/capture/video/linux/video_capture_device_factory_linux.cc @@ -215,17 +215,16 @@ void VideoCaptureDeviceFactoryLinux::GetDeviceDescriptors( device_provider_->GetDeviceDisplayName(unique_id); if (display_name.empty()) display_name = reinterpret_cast<char*>(cap.card); -#if defined(OS_CHROMEOS) device_descriptors->emplace_back( display_name, unique_id, model_id, VideoCaptureApi::LINUX_V4L2_SINGLE_PLANE, VideoCaptureTransportType::OTHER_TRANSPORT, - device_provider_->GetCameraFacing(unique_id, model_id)); +#if defined(OS_CHROMEOS) + device_provider_->GetCameraFacing(unique_id, model_id), #else - device_descriptors->emplace_back( - display_name, unique_id, model_id, - VideoCaptureApi::LINUX_V4L2_SINGLE_PLANE); + VideoFacingMode::MEDIA_VIDEO_FACING_NONE, #endif + IsPanTiltZoomSupported(fd.get())); } } // Since JS doesn't have API to get camera facing, we sort the list to make @@ -254,6 +253,22 @@ int VideoCaptureDeviceFactoryLinux::DoIoctl(int fd, int request, void* argp) { return HANDLE_EINTR(v4l2_->ioctl(fd, request, argp)); } +// Check if the video capture device supports at least one of pan, tilt and zoom +// controls. +bool VideoCaptureDeviceFactoryLinux::IsPanTiltZoomSupported(int fd) { + for (int control_id : {V4L2_CID_PAN_ABSOLUTE, V4L2_CID_TILT_ABSOLUTE, + V4L2_CID_ZOOM_ABSOLUTE}) { + v4l2_queryctrl range = {}; + range.id = control_id; + range.type = V4L2_CTRL_TYPE_INTEGER; + if (DoIoctl(fd, VIDIOC_QUERYCTRL, &range) == 0 && + range.minimum < range.maximum) { + return true; + } + } + return false; +} + bool VideoCaptureDeviceFactoryLinux::HasUsableFormats(int fd, uint32_t capabilities) { if (!(capabilities & V4L2_CAP_VIDEO_CAPTURE)) diff --git a/chromium/media/capture/video/linux/video_capture_device_factory_linux.h b/chromium/media/capture/video/linux/video_capture_device_factory_linux.h index 08c7a831edb..0383924848a 100644 --- a/chromium/media/capture/video/linux/video_capture_device_factory_linux.h +++ b/chromium/media/capture/video/linux/video_capture_device_factory_linux.h @@ -67,6 +67,7 @@ class CAPTURE_EXPORT VideoCaptureDeviceFactoryLinux // Simple wrapper to do HANDLE_EINTR(v4l2_->ioctl(fd, ...)). int DoIoctl(int fd, int request, void* argp); + bool IsPanTiltZoomSupported(int fd); bool HasUsableFormats(int fd, uint32_t capabilities); std::vector<float> GetFrameRateList(int fd, uint32_t fourcc, diff --git a/chromium/media/capture/video/linux/video_capture_device_factory_linux_unittest.cc b/chromium/media/capture/video/linux/video_capture_device_factory_linux_unittest.cc index 88334bb44d5..fd53b895377 100644 --- a/chromium/media/capture/video/linux/video_capture_device_factory_linux_unittest.cc +++ b/chromium/media/capture/video/linux/video_capture_device_factory_linux_unittest.cc @@ -67,7 +67,8 @@ class DescriptorDeviceProvider std::vector<VideoCaptureDeviceDescriptor> descriptors_; }; -class VideoCaptureDeviceFactoryLinuxTest : public ::testing::Test { +class VideoCaptureDeviceFactoryLinuxTest + : public ::testing::TestWithParam<VideoCaptureDeviceDescriptor> { public: VideoCaptureDeviceFactoryLinuxTest() {} ~VideoCaptureDeviceFactoryLinuxTest() override = default; @@ -89,13 +90,11 @@ class VideoCaptureDeviceFactoryLinuxTest : public ::testing::Test { std::unique_ptr<VideoCaptureDeviceFactoryLinux> factory_; }; -TEST_F(VideoCaptureDeviceFactoryLinuxTest, EnumerateSingleFakeV4L2Device) { +TEST_P(VideoCaptureDeviceFactoryLinuxTest, EnumerateSingleFakeV4L2DeviceUsing) { // Setup - const std::string stub_display_name = "Fake Device 0"; - const std::string stub_device_id = "/dev/video0"; - VideoCaptureDeviceDescriptor descriptor(stub_display_name, stub_device_id); + const VideoCaptureDeviceDescriptor& descriptor = GetParam(); fake_device_provider_->AddDevice(descriptor); - fake_v4l2_->AddDevice(stub_device_id, FakeV4L2DeviceConfig(descriptor)); + fake_v4l2_->AddDevice(descriptor.device_id, FakeV4L2DeviceConfig(descriptor)); // Exercise VideoCaptureDeviceDescriptors descriptors; @@ -103,10 +102,27 @@ TEST_F(VideoCaptureDeviceFactoryLinuxTest, EnumerateSingleFakeV4L2Device) { // Verification ASSERT_EQ(1u, descriptors.size()); - ASSERT_EQ(stub_device_id, descriptors[0].device_id); - ASSERT_EQ(stub_display_name, descriptors[0].display_name()); + EXPECT_EQ(descriptor.device_id, descriptors[0].device_id); + EXPECT_EQ(descriptor.display_name(), descriptors[0].display_name()); + EXPECT_EQ(descriptor.pan_tilt_zoom_supported().value(), + descriptors[0].pan_tilt_zoom_supported()); } +INSTANTIATE_TEST_SUITE_P( + All, + VideoCaptureDeviceFactoryLinuxTest, + ::testing::Values( + VideoCaptureDeviceDescriptor("Fake Device 0", + "/dev/video0", + VideoCaptureApi::UNKNOWN, + VideoCaptureTransportType::OTHER_TRANSPORT, + /*pan_tilt_zoom_supported=*/false), + VideoCaptureDeviceDescriptor("Fake Device 0", + "/dev/video0", + VideoCaptureApi::UNKNOWN, + VideoCaptureTransportType::OTHER_TRANSPORT, + /*pan_tilt_zoom_supported=*/true))); + TEST_F(VideoCaptureDeviceFactoryLinuxTest, ReceiveFramesFromSinglePlaneFakeDevice) { // Setup diff --git a/chromium/media/capture/video/mac/video_capture_device_decklink_mac.mm b/chromium/media/capture/video/mac/video_capture_device_decklink_mac.mm index a2378198222..23babdc6821 100644 --- a/chromium/media/capture/video/mac/video_capture_device_decklink_mac.mm +++ b/chromium/media/capture/video/mac/video_capture_device_decklink_mac.mm @@ -412,6 +412,7 @@ void VideoCaptureDeviceDeckLinkMac::EnumerateDevices( JoinDeviceNameAndFormat(device_model_name, format_name); descriptor.capture_api = VideoCaptureApi::MACOSX_DECKLINK; descriptor.transport_type = VideoCaptureTransportType::OTHER_TRANSPORT; + descriptor.set_pan_tilt_zoom_supported(false); device_descriptors->push_back(descriptor); DVLOG(1) << "Blackmagic camera enumerated: " << descriptor.display_name(); diff --git a/chromium/media/capture/video/mac/video_capture_device_factory_mac.mm b/chromium/media/capture/video/mac/video_capture_device_factory_mac.mm index 551ee4ef116..6482d1f1c8a 100644 --- a/chromium/media/capture/video/mac/video_capture_device_factory_mac.mm +++ b/chromium/media/capture/video/mac/video_capture_device_factory_mac.mm @@ -123,7 +123,9 @@ void VideoCaptureDeviceFactoryMac::GetDeviceDescriptors( device_id, capture_api, device_transport_type); VideoCaptureDeviceDescriptor descriptor( [[[capture_devices valueForKey:key] deviceName] UTF8String], device_id, - model_id, capture_api, device_transport_type); + model_id, capture_api, device_transport_type, + VideoFacingMode::MEDIA_VIDEO_FACING_NONE, + /*pan_tilt_zoom_supported=*/false); if (IsDeviceBlacklisted(descriptor)) continue; device_descriptors->push_back(descriptor); diff --git a/chromium/media/capture/video/mac/video_capture_device_factory_mac_unittest.mm b/chromium/media/capture/video/mac/video_capture_device_factory_mac_unittest.mm index 8329266f5e7..8133c4ac671 100644 --- a/chromium/media/capture/video/mac/video_capture_device_factory_mac_unittest.mm +++ b/chromium/media/capture/video/mac/video_capture_device_factory_mac_unittest.mm @@ -46,4 +46,19 @@ TEST(VideoCaptureDeviceFactoryMacTest, ListDevicesAVFoundation) { })); } +TEST(VideoCaptureDeviceFactoryMacTest, ListDevicesWithNoPanTiltZoomSupport) { + RunTestCase(base::BindOnce([]() { + VideoCaptureDeviceFactoryMac video_capture_device_factory; + + VideoCaptureDeviceDescriptors descriptors; + video_capture_device_factory.GetDeviceDescriptors(&descriptors); + if (descriptors.empty()) { + DVLOG(1) << "No camera available. Exiting test."; + return; + } + for (const auto& descriptor : descriptors) + EXPECT_FALSE(descriptor.pan_tilt_zoom_supported().value()); + })); +} + } // namespace media diff --git a/chromium/media/capture/video/video_capture_device.h b/chromium/media/capture/video/video_capture_device.h index 2f0540bbd73..b7143dab16c 100644 --- a/chromium/media/capture/video/video_capture_device.h +++ b/chromium/media/capture/video/video_capture_device.h @@ -21,7 +21,6 @@ #include "base/callback.h" #include "base/files/file.h" -#include "base/logging.h" #include "base/memory/ref_counted.h" #include "base/memory/unsafe_shared_memory_region.h" #include "base/single_thread_task_runner.h" diff --git a/chromium/media/capture/video/video_capture_device_client.cc b/chromium/media/capture/video/video_capture_device_client.cc index 618be0e51ca..43a88b72155 100644 --- a/chromium/media/capture/video/video_capture_device_client.cc +++ b/chromium/media/capture/video/video_capture_device_client.cc @@ -527,10 +527,9 @@ void VideoCaptureDeviceClient::OnIncomingCapturedBufferExt( const VideoFrameMetadata& additional_metadata) { DFAKE_SCOPED_RECURSIVE_LOCK(call_from_producer_); - VideoFrameMetadata metadata; - metadata.MergeMetadataFrom(&additional_metadata); - metadata.SetDouble(VideoFrameMetadata::FRAME_RATE, format.frame_rate); - metadata.SetTimeTicks(VideoFrameMetadata::REFERENCE_TIME, reference_time); + VideoFrameMetadata metadata = additional_metadata; + metadata.frame_rate = format.frame_rate; + metadata.reference_time = reference_time; mojom::VideoFrameInfoPtr info = mojom::VideoFrameInfo::New(); info->timestamp = timestamp; @@ -538,7 +537,7 @@ void VideoCaptureDeviceClient::OnIncomingCapturedBufferExt( info->color_space = color_space; info->coded_size = format.frame_size; info->visible_rect = visible_rect; - info->metadata = metadata.GetInternalValues().Clone(); + info->metadata = metadata; buffer_pool_->HoldForConsumers(buffer.id, 1); receiver_->OnFrameReadyInBuffer( diff --git a/chromium/media/capture/video/video_capture_device_client_unittest.cc b/chromium/media/capture/video/video_capture_device_client_unittest.cc index 3175ba31e69..9bd1d329af9 100644 --- a/chromium/media/capture/video/video_capture_device_client_unittest.cc +++ b/chromium/media/capture/video/video_capture_device_client_unittest.cc @@ -110,7 +110,8 @@ TEST_F(VideoCaptureDeviceClientTest, Minimal) { std::unique_ptr<gfx::GpuMemoryBuffer> buffer = gpu_memory_buffer_manager_->CreateFakeGpuMemoryBuffer( kBufferDimensions, gfx::BufferFormat::YUV_420_BIPLANAR, - gfx::BufferUsage::SCANOUT_CAMERA_READ_WRITE, gpu::kNullSurfaceHandle); + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE, + gpu::kNullSurfaceHandle); { InSequence s; const int expected_buffer_id = 0; diff --git a/chromium/media/capture/video/video_capture_device_descriptor.cc b/chromium/media/capture/video/video_capture_device_descriptor.cc index 81ee65a45b8..8ece28fab36 100644 --- a/chromium/media/capture/video/video_capture_device_descriptor.cc +++ b/chromium/media/capture/video/video_capture_device_descriptor.cc @@ -25,12 +25,14 @@ VideoCaptureDeviceDescriptor::VideoCaptureDeviceDescriptor( const std::string& display_name, const std::string& device_id, VideoCaptureApi capture_api, - VideoCaptureTransportType transport_type) + VideoCaptureTransportType transport_type, + const base::Optional<bool>& pan_tilt_zoom_supported) : device_id(device_id), facing(VideoFacingMode::MEDIA_VIDEO_FACING_NONE), capture_api(capture_api), transport_type(transport_type), - display_name_(TrimDisplayName(display_name)) {} + display_name_(TrimDisplayName(display_name)), + pan_tilt_zoom_supported_(pan_tilt_zoom_supported) {} VideoCaptureDeviceDescriptor::VideoCaptureDeviceDescriptor( const std::string& display_name, @@ -38,13 +40,15 @@ VideoCaptureDeviceDescriptor::VideoCaptureDeviceDescriptor( const std::string& model_id, VideoCaptureApi capture_api, VideoCaptureTransportType transport_type, - VideoFacingMode facing) + VideoFacingMode facing, + const base::Optional<bool>& pan_tilt_zoom_supported) : device_id(device_id), model_id(model_id), facing(facing), capture_api(capture_api), transport_type(transport_type), - display_name_(TrimDisplayName(display_name)) {} + display_name_(TrimDisplayName(display_name)), + pan_tilt_zoom_supported_(pan_tilt_zoom_supported) {} VideoCaptureDeviceDescriptor::~VideoCaptureDeviceDescriptor() = default; diff --git a/chromium/media/capture/video/video_capture_device_descriptor.h b/chromium/media/capture/video/video_capture_device_descriptor.h index 1c3da3f99dc..b2c3eb1caf4 100644 --- a/chromium/media/capture/video/video_capture_device_descriptor.h +++ b/chromium/media/capture/video/video_capture_device_descriptor.h @@ -8,6 +8,7 @@ #include <string> #include <vector> +#include "base/optional.h" #include "media/base/video_facing.h" #include "media/capture/capture_export.h" @@ -55,7 +56,8 @@ struct CAPTURE_EXPORT VideoCaptureDeviceDescriptor { const std::string& device_id, VideoCaptureApi capture_api = VideoCaptureApi::UNKNOWN, VideoCaptureTransportType transport_type = - VideoCaptureTransportType::OTHER_TRANSPORT); + VideoCaptureTransportType::OTHER_TRANSPORT, + const base::Optional<bool>& pan_tilt_zoom_supported = base::nullopt); VideoCaptureDeviceDescriptor( const std::string& display_name, const std::string& device_id, @@ -63,7 +65,8 @@ struct CAPTURE_EXPORT VideoCaptureDeviceDescriptor { VideoCaptureApi capture_api, VideoCaptureTransportType transport_type = VideoCaptureTransportType::OTHER_TRANSPORT, - VideoFacingMode facing = VideoFacingMode::MEDIA_VIDEO_FACING_NONE); + VideoFacingMode facing = VideoFacingMode::MEDIA_VIDEO_FACING_NONE, + const base::Optional<bool>& pan_tilt_zoom_supported = base::nullopt); VideoCaptureDeviceDescriptor(const VideoCaptureDeviceDescriptor& other); ~VideoCaptureDeviceDescriptor(); @@ -83,6 +86,13 @@ struct CAPTURE_EXPORT VideoCaptureDeviceDescriptor { const std::string& display_name() const { return display_name_; } void set_display_name(const std::string& name); + const base::Optional<bool>& pan_tilt_zoom_supported() const { + return pan_tilt_zoom_supported_; + } + void set_pan_tilt_zoom_supported(bool supported) { + pan_tilt_zoom_supported_ = supported; + } + std::string device_id; // A unique hardware identifier of the capture device. // It is of the form "[vid]:[pid]" when a USB device is detected, and empty @@ -96,6 +106,7 @@ struct CAPTURE_EXPORT VideoCaptureDeviceDescriptor { private: std::string display_name_; // Name that is intended for display in the UI + base::Optional<bool> pan_tilt_zoom_supported_; }; using VideoCaptureDeviceDescriptors = std::vector<VideoCaptureDeviceDescriptor>; diff --git a/chromium/media/capture/video/win/video_capture_device_factory_win.cc b/chromium/media/capture/video/win/video_capture_device_factory_win.cc index 64474252226..a79f4d7761e 100644 --- a/chromium/media/capture/video/win/video_capture_device_factory_win.cc +++ b/chromium/media/capture/video/win/video_capture_device_factory_win.cc @@ -22,6 +22,7 @@ #include "base/stl_util.h" #include "base/strings/string_util.h" #include "base/strings/sys_string_conversions.h" +#include "base/threading/scoped_thread_priority.h" #include "base/win/core_winrt_util.h" #include "base/win/scoped_co_mem.h" #include "base/win/scoped_variant.h" @@ -53,13 +54,13 @@ const size_t kVidPidSize = 4; // Avoid enumerating and/or using certain devices due to they provoking crashes // or any other reason (http://crbug.com/378494). This enum is defined for the // purposes of UMA collection. Existing entries cannot be removed. -enum BlacklistedCameraNames { - BLACKLISTED_CAMERA_GOOGLE_CAMERA_ADAPTER = 0, - BLACKLISTED_CAMERA_IP_CAMERA = 1, - BLACKLISTED_CAMERA_CYBERLINK_WEBCAM_SPLITTER = 2, - BLACKLISTED_CAMERA_EPOCCAM = 3, +enum BlockedCameraNames { + BLOCKED_CAMERA_GOOGLE_CAMERA_ADAPTER = 0, + BLOCKED_CAMERA_IP_CAMERA = 1, + BLOCKED_CAMERA_CYBERLINK_WEBCAM_SPLITTER = 2, + BLOCKED_CAMERA_EPOCCAM = 3, // This one must be last, and equal to the previous enumerated value. - BLACKLISTED_CAMERA_MAX = BLACKLISTED_CAMERA_EPOCCAM, + BLOCKED_CAMERA_MAX = BLOCKED_CAMERA_EPOCCAM, }; #define UWP_ENUM_ERROR_HANDLER(hr, err_log) \ @@ -67,21 +68,23 @@ enum BlacklistedCameraNames { origin_task_runner_->PostTask(FROM_HERE, \ base::BindOnce(device_info_callback, nullptr)) -// Blacklisted devices are identified by a characteristic prefix of the name. +// Blocked devices are identified by a characteristic prefix of the name. // This prefix is used case-insensitively. This list must be kept in sync with -// |BlacklistedCameraNames|. -const char* const kBlacklistedCameraNames[] = { +// |BlockedCameraNames|. +const char* const kBlockedCameraNames[] = { // Name of a fake DirectShow filter on computers with GTalk installed. "Google Camera Adapter", // The following software WebCams cause crashes. - "IP Camera [JPEG/MJPEG]", "CyberLink Webcam Splitter", "EpocCam", + "IP Camera [JPEG/MJPEG]", + "CyberLink Webcam Splitter", + "EpocCam", }; -static_assert(base::size(kBlacklistedCameraNames) == BLACKLISTED_CAMERA_MAX + 1, - "kBlacklistedCameraNames should be same size as " - "BlacklistedCameraNames enum"); +static_assert(base::size(kBlockedCameraNames) == BLOCKED_CAMERA_MAX + 1, + "kBlockedCameraNames should be same size as " + "BlockedCameraNames enum"); // Use this list only for USB webcams. -const char* const kModelIdsBlacklistedForMediaFoundation[] = { +const char* const kModelIdsBlockedForMediaFoundation[] = { // Devices using Empia 2860 or 2820 chips, see https://crbug.com/849636. "eb1a:2860", "eb1a:2820", "1ce6:2820", // Elgato HD60 Pro @@ -105,7 +108,7 @@ const char* const kModelIdsBlacklistedForMediaFoundation[] = { "0bda:57f2"}; // Use this list only for non-USB webcams. -const char* const kDisplayNamesBlacklistedForMediaFoundation[] = { +const char* const kDisplayNamesBlockedForMediaFoundation[] = { // VMware Virtual Webcams cause hangs when there is no physical Webcam. // See https://crbug.com/1044974. "VMware Virtual Webcam"}; @@ -128,20 +131,18 @@ GetMFAttributes() { return *mf_attributes; } -bool IsDeviceBlacklistedForQueryingDetailedFrameRates( +bool IsDeviceBlockedForQueryingDetailedFrameRates( const std::string& display_name) { return display_name.find("WebcamMax") != std::string::npos; } -bool IsDeviceBlacklistedForMediaFoundationByModelId( - const std::string& model_id) { - return base::Contains(kModelIdsBlacklistedForMediaFoundation, model_id); +bool IsDeviceBlockedForMediaFoundationByModelId(const std::string& model_id) { + return base::Contains(kModelIdsBlockedForMediaFoundation, model_id); } -bool IsDeviceBlacklistedForMediaFoundationByDisplayName( +bool IsDeviceBlockedForMediaFoundationByDisplayName( const std::string& display_name) { - return base::Contains(kDisplayNamesBlacklistedForMediaFoundation, - display_name); + return base::Contains(kDisplayNamesBlockedForMediaFoundation, display_name); } bool LoadMediaFoundationDlls() { @@ -150,6 +151,10 @@ bool LoadMediaFoundationDlls() { L"%WINDIR%\\system32\\mfreadwrite.dll", L"%WINDIR%\\system32\\MFCaptureEngine.dll"}; + // Mitigate the issues caused by loading DLLs on a background thread + // (http://crbug/973868). + SCOPED_MAY_LOAD_LIBRARY_AT_BACKGROUND_PRIORITY_REPEATEDLY(); + for (const wchar_t* kMfDLL : kMfDLLs) { wchar_t path[MAX_PATH] = {0}; ExpandEnvironmentStringsW(kMfDLL, path, base::size(path)); @@ -199,15 +204,15 @@ bool CreateVideoCaptureDeviceMediaFoundation(const Descriptor& descriptor, return SUCCEEDED(MFCreateDeviceSource(attributes.Get(), source)); } -bool IsDeviceBlackListed(const std::string& name) { - DCHECK_EQ(BLACKLISTED_CAMERA_MAX + 1, - static_cast<int>(base::size(kBlacklistedCameraNames))); - for (size_t i = 0; i < base::size(kBlacklistedCameraNames); ++i) { - if (base::StartsWith(name, kBlacklistedCameraNames[i], +bool IsDeviceBlocked(const std::string& name) { + DCHECK_EQ(BLOCKED_CAMERA_MAX + 1, + static_cast<int>(base::size(kBlockedCameraNames))); + for (size_t i = 0; i < base::size(kBlockedCameraNames); ++i) { + if (base::StartsWith(name, kBlockedCameraNames[i], base::CompareCase::INSENSITIVE_ASCII)) { - DVLOG(1) << "Enumerated blacklisted device: " << name; + DVLOG(1) << "Enumerated blocked device: " << name; UMA_HISTOGRAM_ENUMERATION("Media.VideoCapture.BlacklistedDevice", i, - BLACKLISTED_CAMERA_MAX + 1); + BLOCKED_CAMERA_MAX + 1); return true; } } @@ -235,6 +240,10 @@ std::string GetDeviceModelId(const std::string& device_id) { } HRESULT EnumerateDirectShowDevices(IEnumMoniker** enum_moniker) { + // Mitigate the issues caused by loading DLLs on a background thread + // (http://crbug/973868). + SCOPED_MAY_LOAD_LIBRARY_AT_BACKGROUND_PRIORITY(); + ComPtr<ICreateDevEnum> dev_enum; HRESULT hr = ::CoCreateInstance(CLSID_SystemDeviceEnum, NULL, CLSCTX_INPROC, IID_PPV_ARGS(&dev_enum)); @@ -272,8 +281,7 @@ void GetDeviceSupportedFormatsDirectShow(const Descriptor& descriptor, DVLOG(1) << "GetDeviceSupportedFormatsDirectShow for " << descriptor.display_name(); bool query_detailed_frame_rates = - !IsDeviceBlacklistedForQueryingDetailedFrameRates( - descriptor.display_name()); + !IsDeviceBlockedForQueryingDetailedFrameRates(descriptor.display_name()); CapabilityList capability_list; VideoCaptureDeviceWin::GetDeviceCapabilityList( descriptor.device_id, query_detailed_frame_rates, &capability_list); @@ -662,9 +670,8 @@ void VideoCaptureDeviceFactoryWin::GetDeviceDescriptorsMediaFoundation( const std::string model_id = GetDeviceModelId(device_id); const std::string display_name = base::SysWideToUTF8(std::wstring(name, name_size)); - if (IsDeviceBlacklistedForMediaFoundationByModelId(model_id) || - IsDeviceBlacklistedForMediaFoundationByDisplayName( - display_name)) { + if (IsDeviceBlockedForMediaFoundationByModelId(model_id) || + IsDeviceBlockedForMediaFoundationByDisplayName(display_name)) { continue; } if (list_was_empty || @@ -763,7 +770,7 @@ void VideoCaptureDeviceFactoryWin::GetDeviceDescriptorsDirectShow( continue; const std::string device_name(base::SysWideToUTF8(V_BSTR(name.ptr()))); - if (IsDeviceBlackListed(device_name)) + if (IsDeviceBlocked(device_name)) continue; name.Reset(); diff --git a/chromium/media/capture/video/win/video_capture_device_mf_win.cc b/chromium/media/capture/video/win/video_capture_device_mf_win.cc index c432b210efd..bcadd51789d 100644 --- a/chromium/media/capture/video/win/video_capture_device_mf_win.cc +++ b/chromium/media/capture/video/win/video_capture_device_mf_win.cc @@ -125,6 +125,76 @@ class MFPhotoCallback final DISALLOW_COPY_AND_ASSIGN(MFPhotoCallback); }; +// Locks the given buffer using the fastest supported method when constructed, +// and automatically unlocks the buffer when destroyed. +class ScopedBufferLock { + public: + ScopedBufferLock(ComPtr<IMFMediaBuffer> buffer) : buffer_(std::move(buffer)) { + if (FAILED(buffer_.As(&buffer_2d_))) { + LockSlow(); + return; + } + // Try lock methods from fastest to slowest: Lock2DSize(), then Lock2D(), + // then finally LockSlow(). + if ((Lock2DSize() || Lock2D()) && !UnlockedNoncontiguousBuffer()) + return; + // Fall back to LockSlow() if 2D buffer was unsupported or noncontiguous. + buffer_2d_ = nullptr; + LockSlow(); + } + + // Unlocks |buffer_2d_| and returns true if |buffer_2d_| is non-contiguous or + // has negative pitch. If |buffer_2d_| is contiguous with positive pitch, + // i.e., the buffer format that the surrounding code expects, returns false. + bool UnlockedNoncontiguousBuffer() { + BOOL is_contiguous; + if (pitch_ > 0 && + SUCCEEDED(buffer_2d_->IsContiguousFormat(&is_contiguous)) && + is_contiguous && + (length_ || SUCCEEDED(buffer_2d_->GetContiguousLength(&length_)))) { + return false; + } + buffer_2d_->Unlock2D(); + return true; + } + + bool Lock2DSize() { + ComPtr<IMF2DBuffer2> buffer_2d_2; + if (FAILED(buffer_.As(&buffer_2d_2))) + return false; + BYTE* data_start; + return SUCCEEDED(buffer_2d_2->Lock2DSize(MF2DBuffer_LockFlags_Read, &data_, + &pitch_, &data_start, &length_)); + } + + bool Lock2D() { return SUCCEEDED(buffer_2d_->Lock2D(&data_, &pitch_)); } + + void LockSlow() { + DWORD max_length = 0; + buffer_->Lock(&data_, &max_length, &length_); + } + + ~ScopedBufferLock() { + if (buffer_2d_) + buffer_2d_->Unlock2D(); + else + buffer_->Unlock(); + } + + ScopedBufferLock(const ScopedBufferLock&) = delete; + ScopedBufferLock& operator=(const ScopedBufferLock&) = delete; + + BYTE* data() const { return data_; } + DWORD length() const { return length_; } + + private: + ComPtr<IMFMediaBuffer> buffer_; + ComPtr<IMF2DBuffer> buffer_2d_; + BYTE* data_ = nullptr; + DWORD length_ = 0; + LONG pitch_ = 0; +}; + scoped_refptr<IMFCaptureEngineOnSampleCallback> CreateMFPhotoCallback( VideoCaptureDevice::TakePhotoCallback callback, VideoCaptureFormat format) { @@ -414,11 +484,19 @@ class MFVideoCallback final } IFACEMETHODIMP OnEvent(IMFMediaEvent* media_event) override { + base::AutoLock lock(lock_); + if (!observer_) { + return S_OK; + } observer_->OnEvent(media_event); return S_OK; } IFACEMETHODIMP OnSample(IMFSample* sample) override { + base::AutoLock lock(lock_); + if (!observer_) { + return S_OK; + } if (!sample) { observer_->OnFrameDropped( VideoCaptureFrameDropReason::kWinMediaFoundationReceivedSampleIsNull); @@ -438,60 +516,16 @@ class MFVideoCallback final ComPtr<IMFMediaBuffer> buffer; sample->GetBufferByIndex(i, &buffer); if (buffer) { - // Lock the buffer using the fastest method that it supports. The - // Lock2DSize() method is faster than Lock2D(), which is faster than - // Lock(). - DWORD length = 0; - BYTE* data = nullptr; - ComPtr<IMF2DBuffer> buffer_2d; - if (SUCCEEDED(buffer.As(&buffer_2d))) { - HRESULT lock_result; - BYTE* scanline_0 = nullptr; - LONG pitch = 0; - ComPtr<IMF2DBuffer2> buffer_2d_2; - if (SUCCEEDED(buffer.As(&buffer_2d_2))) { - BYTE* data_start; - lock_result = - buffer_2d_2->Lock2DSize(MF2DBuffer_LockFlags_Read, &scanline_0, - &pitch, &data_start, &length); - } else { - lock_result = buffer_2d->Lock2D(&scanline_0, &pitch); - } - if (SUCCEEDED(lock_result)) { - // Use |buffer_2d| only if it is contiguous and has positive pitch. - BOOL is_contiguous; - if (pitch > 0 && - SUCCEEDED(buffer_2d->IsContiguousFormat(&is_contiguous)) && - is_contiguous && - (length || - SUCCEEDED(buffer_2d->GetContiguousLength(&length)))) { - data = scanline_0; - } else { - buffer_2d->Unlock2D(); - } - } - } - if (!data) { - // If the faster methods fail, fall back to Lock to lock the buffer. - buffer_2d = nullptr; - DWORD max_length = 0; - buffer->Lock(&data, &max_length, &length); - } - - if (data) { - observer_->OnIncomingCapturedData(data, length, reference_time, - timestamp); + ScopedBufferLock locked_buffer(buffer); + if (locked_buffer.data()) { + observer_->OnIncomingCapturedData(locked_buffer.data(), + locked_buffer.length(), + reference_time, timestamp); } else { observer_->OnFrameDropped( VideoCaptureFrameDropReason:: kWinMediaFoundationLockingBufferDelieveredNullptr); } - - if (buffer_2d) - buffer_2d->Unlock2D(); - else - buffer->Unlock(); - } else { observer_->OnFrameDropped( VideoCaptureFrameDropReason:: @@ -501,10 +535,18 @@ class MFVideoCallback final return S_OK; } + void Shutdown() { + base::AutoLock lock(lock_); + observer_ = nullptr; + } + private: friend class base::RefCountedThreadSafe<MFVideoCallback>; ~MFVideoCallback() {} - VideoCaptureDeviceMFWin* observer_; + + // Protects access to |observer_|. + base::Lock lock_; + VideoCaptureDeviceMFWin* observer_ GUARDED_BY(lock_); }; // static @@ -657,7 +699,12 @@ VideoCaptureDeviceMFWin::VideoCaptureDeviceMFWin( has_sent_on_started_to_client_(false), exposure_mode_manual_(false), focus_mode_manual_(false), - white_balance_mode_manual_(false) { + white_balance_mode_manual_(false), + capture_initialize_(base::WaitableEvent::ResetPolicy::AUTOMATIC, + base::WaitableEvent::InitialState::NOT_SIGNALED), + // We never want to reset |capture_error_|. + capture_error_(base::WaitableEvent::ResetPolicy::MANUAL, + base::WaitableEvent::InitialState::NOT_SIGNALED) { DETACH_FROM_SEQUENCE(sequence_checker_); } @@ -673,6 +720,9 @@ VideoCaptureDeviceMFWin::~VideoCaptureDeviceMFWin() { : false); } } + if (video_callback_) { + video_callback_->Shutdown(); + } } bool VideoCaptureDeviceMFWin::Init() { @@ -705,6 +755,13 @@ bool VideoCaptureDeviceMFWin::Init() { LogError(FROM_HERE, hr); return false; } + + hr = WaitOnCaptureEvent(MF_CAPTURE_ENGINE_INITIALIZED); + if (FAILED(hr)) { + LogError(FROM_HERE, hr); + return false; + } + is_initialized_ = true; return true; } @@ -1294,7 +1351,21 @@ void VideoCaptureDeviceMFWin::OnEvent(IMFMediaEvent* media_event) { base::AutoLock lock(lock_); HRESULT hr; + GUID capture_event_guid = GUID_NULL; + media_event->GetStatus(&hr); + media_event->GetExtendedType(&capture_event_guid); + // TODO(http://crbug.com/1093521): Add cases for Start + // MF_CAPTURE_ENGINE_PREVIEW_STARTED and MF_CAPTURE_ENGINE_PREVIEW_STOPPED + // When MF_CAPTURE_ENGINE_ERROR is returned the captureengine object is no + // longer valid. + if (capture_event_guid == MF_CAPTURE_ENGINE_ERROR || FAILED(hr)) { + capture_error_.Signal(); + // There should always be a valid error + hr = SUCCEEDED(hr) ? E_UNEXPECTED : hr; + } else if (capture_event_guid == MF_CAPTURE_ENGINE_INITIALIZED) { + capture_initialize_.Signal(); + } if (FAILED(hr)) OnError(VideoCaptureError::kWinMediaFoundationGetMediaEventStatusFailed, @@ -1324,4 +1395,35 @@ void VideoCaptureDeviceMFWin::SendOnStartedIfNotYetSent() { client_->OnStarted(); } +HRESULT VideoCaptureDeviceMFWin::WaitOnCaptureEvent(GUID capture_event_guid) { + HRESULT hr = S_OK; + HANDLE events[] = {nullptr, capture_error_.handle()}; + + // TODO(http://crbug.com/1093521): Add cases for Start + // MF_CAPTURE_ENGINE_PREVIEW_STARTED and MF_CAPTURE_ENGINE_PREVIEW_STOPPED + if (capture_event_guid == MF_CAPTURE_ENGINE_INITIALIZED) { + events[0] = capture_initialize_.handle(); + } else { + // no registered event handle for the event requested + hr = E_NOTIMPL; + LogError(FROM_HERE, hr); + return hr; + } + + DWORD wait_result = + ::WaitForMultipleObjects(base::size(events), events, FALSE, INFINITE); + switch (wait_result) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + hr = HRESULT_FROM_WIN32(::GetLastError()); + LogError(FROM_HERE, hr); + break; + default: + hr = E_UNEXPECTED; + LogError(FROM_HERE, hr); + break; + } + return hr; +} } // namespace media diff --git a/chromium/media/capture/video/win/video_capture_device_mf_win.h b/chromium/media/capture/video/win/video_capture_device_mf_win.h index e5ecd1250ba..a3d0e7db763 100644 --- a/chromium/media/capture/video/win/video_capture_device_mf_win.h +++ b/chromium/media/capture/video/win/video_capture_device_mf_win.h @@ -117,6 +117,7 @@ class CAPTURE_EXPORT VideoCaptureDeviceMFWin : public VideoCaptureDevice { const base::Location& from_here, const char* message); void SendOnStartedIfNotYetSent(); + HRESULT WaitOnCaptureEvent(GUID capture_event_guid); VideoFacingMode facing_mode_; CreateMFPhotoCallbackCB create_mf_photo_callback_; @@ -145,6 +146,8 @@ class CAPTURE_EXPORT VideoCaptureDeviceMFWin : public VideoCaptureDevice { bool focus_mode_manual_; bool white_balance_mode_manual_; base::queue<TakePhotoCallback> video_stream_take_photo_callbacks_; + base::WaitableEvent capture_initialize_; + base::WaitableEvent capture_error_; SEQUENCE_CHECKER(sequence_checker_); diff --git a/chromium/media/capture/video/win/video_capture_device_mf_win_unittest.cc b/chromium/media/capture/video/win/video_capture_device_mf_win_unittest.cc index e8575b891ef..5307405382d 100644 --- a/chromium/media/capture/video/win/video_capture_device_mf_win_unittest.cc +++ b/chromium/media/capture/video/win/video_capture_device_mf_win_unittest.cc @@ -8,6 +8,8 @@ #include <wincodec.h> #include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/test/task_environment.h" #include "media/capture/video/win/sink_filter_win.h" #include "media/capture/video/win/video_capture_device_factory_win.h" #include "media/capture/video/win/video_capture_device_mf_win.h" @@ -413,11 +415,34 @@ class MockMFCaptureEngine EXPECT_TRUE(pAttributes); EXPECT_TRUE(pVideoSource); event_callback = pEventCallback; - OnCorrectInitialize(); + OnCorrectInitializeQueued(); + + ON_CALL(*this, OnInitStatus).WillByDefault(Return(S_OK)); + ON_CALL(*this, OnInitEventGuid) + .WillByDefault(Return(MF_CAPTURE_ENGINE_INITIALIZED)); + // HW Cameras usually add about 500ms latency on init + ON_CALL(*this, InitEventDelay) + .WillByDefault(Return(base::TimeDelta::FromMilliseconds(500))); + + base::TimeDelta event_delay = InitEventDelay(); + + base::ThreadPool::PostDelayedTask( + FROM_HERE, + base::BindOnce(&MockMFCaptureEngine::FireCaptureEvent, this, + OnInitEventGuid(), OnInitStatus()), + event_delay); + // if zero is passed ensure event fires before wait starts + if (event_delay == base::TimeDelta::FromMilliseconds(0)) { + base::PlatformThread::Sleep(base::TimeDelta::FromMilliseconds(200)); + } + return S_OK; } - MOCK_METHOD0(OnCorrectInitialize, void(void)); + MOCK_METHOD0(OnCorrectInitializeQueued, void(void)); + MOCK_METHOD0(OnInitEventGuid, GUID(void)); + MOCK_METHOD0(OnInitStatus, HRESULT(void)); + MOCK_METHOD0(InitEventDelay, base::TimeDelta(void)); IFACEMETHODIMP StartPreview(void) override { OnStartPreview(); @@ -456,8 +481,14 @@ class MockMFCaptureEngine } MOCK_METHOD0(DoGetSource, IMFCaptureSource*()); + void FireCaptureEvent(GUID event, HRESULT hrStatus) { + ComPtr<IMFMediaEvent> captureEvent; + MFCreateMediaEvent(MEExtendedType, event, hrStatus, nullptr, &captureEvent); + if (event_callback) { + event_callback->OnEvent(captureEvent.Get()); + } + } scoped_refptr<IMFCaptureEngineOnEventCallback> event_callback; - private: friend class base::RefCountedThreadSafe<MockMFCaptureEngine>; virtual ~MockMFCaptureEngine() = default; @@ -872,7 +903,7 @@ class VideoCaptureDeviceMFWinTest : public ::testing::Test { device_->set_max_retry_count_for_testing(3); device_->set_retry_delay_in_ms_for_testing(1); - EXPECT_CALL(*(engine_.Get()), OnCorrectInitialize()); + EXPECT_CALL(*(engine_.Get()), OnCorrectInitializeQueued()); EXPECT_TRUE(device_->Init()); EXPECT_CALL(*(engine_.Get()), DoGetSource()) .WillRepeatedly(Invoke([this]() { @@ -1079,6 +1110,7 @@ class VideoCaptureDeviceMFWinTest : public ::testing::Test { scoped_refptr<MockMFCaptureSource> capture_source_; scoped_refptr<MockCapturePreviewSink> capture_preview_sink_; + base::test::TaskEnvironment task_environment_; private: const bool media_foundation_supported_; @@ -1118,6 +1150,91 @@ TEST_F(VideoCaptureDeviceMFWinTest, CallClientOnErrorMediaEvent) { engine_->event_callback->OnEvent(media_event_error.get()); } +// Expects Init to fail due to OnError() event +TEST_F(VideoCaptureDeviceMFWinTest, CallClientOnErrorDurringInit) { + if (ShouldSkipTest()) + return; + + VideoCaptureDeviceDescriptor descriptor = VideoCaptureDeviceDescriptor(); + Microsoft::WRL::ComPtr<MockMFMediaSource> media_source = + new MockMFMediaSource(); + Microsoft::WRL::ComPtr<MockMFCaptureEngine> engine = + new MockMFCaptureEngine(); + std::unique_ptr<VideoCaptureDeviceMFWin> device = + std::make_unique<VideoCaptureDeviceMFWin>(descriptor, media_source, + engine); + + EXPECT_CALL(*(engine.Get()), OnInitEventGuid).WillOnce([]() { + return MF_CAPTURE_ENGINE_INITIALIZED; + }); + // E_ACCESSDENIED is thrown if application is denied access in settings UI + EXPECT_CALL(*(engine.Get()), OnInitStatus).WillOnce([]() { + return E_ACCESSDENIED; + }); + + EXPECT_CALL(*(engine.Get()), OnCorrectInitializeQueued()); + + EXPECT_FALSE(device->Init()); +} + +// Expects Init to succeed but MF_CAPTURE_ENGINE_INITIALIZED fired before +// WaitOnCaptureEvent is called. +TEST_F(VideoCaptureDeviceMFWinTest, CallClientOnFireCaptureEngineInitEarly) { + if (ShouldSkipTest()) + return; + + VideoCaptureDeviceDescriptor descriptor = VideoCaptureDeviceDescriptor(); + Microsoft::WRL::ComPtr<MockMFMediaSource> media_source = + new MockMFMediaSource(); + Microsoft::WRL::ComPtr<MockMFCaptureEngine> engine = + new MockMFCaptureEngine(); + std::unique_ptr<VideoCaptureDeviceMFWin> device = + std::make_unique<VideoCaptureDeviceMFWin>(descriptor, media_source, + engine); + + EXPECT_CALL(*(engine.Get()), OnInitEventGuid).WillOnce([]() { + return MF_CAPTURE_ENGINE_INITIALIZED; + }); + EXPECT_CALL(*(engine.Get()), InitEventDelay).WillOnce([]() { + return base::TimeDelta::FromMilliseconds(0); + }); + + EXPECT_CALL(*(engine.Get()), OnCorrectInitializeQueued()); + + EXPECT_TRUE(device->Init()); +} + +// Send MFVideoCallback::OnEvent when VideoCaptureDeviceMFWin has been destroyed +TEST_F(VideoCaptureDeviceMFWinTest, + SendMFVideoCallbackAfterVideoCaptureDeviceMFWinDestructor) { + if (ShouldSkipTest()) + return; + + VideoCaptureDeviceDescriptor descriptor = VideoCaptureDeviceDescriptor(); + Microsoft::WRL::ComPtr<MockMFMediaSource> media_source = + new MockMFMediaSource(); + Microsoft::WRL::ComPtr<MockMFCaptureEngine> engine = + new MockMFCaptureEngine(); + std::unique_ptr<VideoCaptureDeviceMFWin> device = + std::make_unique<VideoCaptureDeviceMFWin>(descriptor, media_source, + engine); + + EXPECT_CALL(*(engine.Get()), OnInitEventGuid).WillOnce([]() { + return MF_CAPTURE_ENGINE_INITIALIZED; + }); + + EXPECT_CALL(*(engine.Get()), OnCorrectInitializeQueued()); + + EXPECT_TRUE(device->Init()); + + // Force ~VideoCaptureDeviceMFWin() which will invalidate + // MFVideoCallback::observer_ + device.reset(); + // Send event to MFVideoCallback::OnEvent + engine->FireCaptureEvent(MF_CAPTURE_ENGINE_ERROR, + MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED); +} + // Allocates device with flaky methods failing with MF_E_INVALIDREQUEST and // expects the device to retry and start correctly TEST_F(VideoCaptureDeviceMFWinTest, AllocateAndStartWithFlakyInvalidRequest) { |