diff options
Diffstat (limited to 'chromium/chrome/renderer')
114 files changed, 1961 insertions, 4566 deletions
diff --git a/chromium/chrome/renderer/BUILD.gn b/chromium/chrome/renderer/BUILD.gn index 2ea489762ef..22e0426b61a 100644 --- a/chromium/chrome/renderer/BUILD.gn +++ b/chromium/chrome/renderer/BUILD.gn @@ -37,7 +37,10 @@ grit("resources") { "//url/mojom:url_mojom_origin_js", ] if (is_chromeos) { - deps += [ "//chromeos/services/ime/public/mojom:mojom_js" ] + deps += [ + "//chromeos/services/ime/public/mojom:mojom_js", + "//chromeos/services/tts/public/mojom:mojom_js", + ] } } @@ -57,6 +60,12 @@ static_library("renderer") { "chrome_render_thread_observer.h", "custom_menu_commands.h", "instant_restricted_id_cache.h", + "lite_video/lite_video_hint_agent.cc", + "lite_video/lite_video_hint_agent.h", + "lite_video/lite_video_url_loader_throttle.cc", + "lite_video/lite_video_url_loader_throttle.h", + "lite_video/lite_video_util.cc", + "lite_video/lite_video_util.h", "loadtimes_extension_bindings.cc", "loadtimes_extension_bindings.h", "media/chrome_key_systems.cc", @@ -251,8 +260,6 @@ static_library("renderer") { ] if (safe_browsing_mode == 1) { sources += [ - "safe_browsing/feature_extractor_clock.cc", - "safe_browsing/feature_extractor_clock.h", "safe_browsing/features.cc", "safe_browsing/features.h", "safe_browsing/murmurhash3_util.cc", @@ -311,26 +318,6 @@ static_library("renderer") { "extensions/sync_file_system_custom_bindings.h", "extensions/tabs_hooks_delegate.cc", "extensions/tabs_hooks_delegate.h", - "media/cast_ipc_dispatcher.cc", - "media/cast_ipc_dispatcher.h", - "media/cast_receiver_audio_valve.cc", - "media/cast_receiver_audio_valve.h", - "media/cast_receiver_session.cc", - "media/cast_receiver_session.h", - "media/cast_receiver_session_delegate.cc", - "media/cast_receiver_session_delegate.h", - "media/cast_rtp_stream.cc", - "media/cast_rtp_stream.h", - "media/cast_session.cc", - "media/cast_session.h", - "media/cast_session_delegate.cc", - "media/cast_session_delegate.h", - "media/cast_threads.cc", - "media/cast_threads.h", - "media/cast_transport_ipc.cc", - "media/cast_transport_ipc.h", - "media/cast_udp_transport.cc", - "media/cast_udp_transport.h", ] deps += [ # TODO(hclam): See crbug.com/298380 for details. @@ -346,12 +333,6 @@ static_library("renderer") { ] public_deps += [ "//ipc" ] } - if (enable_extensions) { - sources += [ - "extensions/cast_streaming_native_handler.cc", - "extensions/cast_streaming_native_handler.h", - ] - } if (enable_spellcheck) { deps += [ "//components/spellcheck/renderer:renderer" ] } @@ -432,7 +413,7 @@ static_library("renderer") { ] } - if (is_linux) { + if (is_chromeos) { sources += [ "performance_manager/mechanisms/tcmalloc_tunables_impl.cc", "performance_manager/mechanisms/tcmalloc_tunables_impl.h", @@ -449,8 +430,6 @@ static_library("test_support") { sources = [ "chrome_mock_render_thread.cc", "chrome_mock_render_thread.h", - "safe_browsing/mock_feature_extractor_clock.cc", - "safe_browsing/mock_feature_extractor_clock.h", "safe_browsing/test_utils.cc", "safe_browsing/test_utils.h", ] @@ -463,11 +442,4 @@ static_library("test_support") { "//testing/gmock", "//testing/gtest", ] - - if (is_android) { - sources -= [ - "safe_browsing/mock_feature_extractor_clock.cc", - "safe_browsing/mock_feature_extractor_clock.h", - ] - } } diff --git a/chromium/chrome/renderer/DEPS b/chromium/chrome/renderer/DEPS index b07362408fd..76e46eb27a0 100644 --- a/chromium/chrome/renderer/DEPS +++ b/chromium/chrome/renderer/DEPS @@ -34,6 +34,7 @@ include_rules = [ "+components/password_manager/core/common", "+components/pdf/renderer", "+components/plugins/renderer", + "+components/prerender", "+components/printing/common", "+components/printing/renderer", "+components/rappor/public/mojom", diff --git a/chromium/chrome/renderer/autofill/autofill_renderer_browsertest.cc b/chromium/chrome/renderer/autofill/autofill_renderer_browsertest.cc index f83951ee485..41fba40e3b5 100644 --- a/chromium/chrome/renderer/autofill/autofill_renderer_browsertest.cc +++ b/chromium/chrome/renderer/autofill/autofill_renderer_browsertest.cc @@ -119,9 +119,6 @@ class FakeContentAutofillDriver : public mojom::AutofillDriver { void DidEndTextFieldEditing() override {} - void SetDataList(const std::vector<base::string16>& values, - const std::vector<base::string16>& labels) override {} - void SelectFieldOptionsDidChange(const autofill::FormData& form) override {} // Records whether TextFieldDidChange() get called. diff --git a/chromium/chrome/renderer/autofill/form_autocomplete_browsertest.cc b/chromium/chrome/renderer/autofill/form_autocomplete_browsertest.cc index 4e49269be5d..56033d8077d 100644 --- a/chromium/chrome/renderer/autofill/form_autocomplete_browsertest.cc +++ b/chromium/chrome/renderer/autofill/form_autocomplete_browsertest.cc @@ -109,9 +109,6 @@ class FakeContentAutofillDriver : public mojom::AutofillDriver { void DidEndTextFieldEditing() override {} - void SetDataList(const std::vector<base::string16>& values, - const std::vector<base::string16>& labels) override {} - void SelectFieldOptionsDidChange(const autofill::FormData& form) override {} // Records whether FocusNoLongerOnForm() get called. @@ -966,8 +963,8 @@ TEST_F(FormAutocompleteTest, FormSubmittedBySameDocumentNavigation) { ExecuteJavaScriptForTests(hide_elements.c_str()); // Simulate same document navigation. - autofill_agent_->form_tracker_for_testing()->DidCommitProvisionalLoad( - true /*is_same_document_navigation*/, ui::PAGE_TRANSITION_LINK); + autofill_agent_->form_tracker_for_testing() + ->DidFinishSameDocumentNavigation(); base::RunLoop().RunUntilIdle(); VerifyReceivedAddressRendererMessages( diff --git a/chromium/chrome/renderer/autofill/password_autofill_agent_browsertest.cc b/chromium/chrome/renderer/autofill/password_autofill_agent_browsertest.cc index 006032ba871..92de649dd28 100644 --- a/chromium/chrome/renderer/autofill/password_autofill_agent_browsertest.cc +++ b/chromium/chrome/renderer/autofill/password_autofill_agent_browsertest.cc @@ -381,7 +381,7 @@ class PasswordAutofillAgentTest : public ChromeRenderViewTest { // We need to set the origin so it matches the frame URL and the action so // it matches the form action, otherwise we won't autocomplete. - UpdateOriginForHTML(kFormHTML); + UpdateUrlForHTML(kFormHTML); fill_data_.action = GURL("http://www.bidule.com"); LoadHTML(kFormHTML); @@ -441,9 +441,9 @@ class PasswordAutofillAgentTest : public ChromeRenderViewTest { switches::kShowAutofillSignatures); } - void UpdateOriginForHTML(const std::string& html) { - std::string origin = "data:text/html;charset=utf-8," + html; - fill_data_.origin = GURL(origin); + void UpdateUrlForHTML(const std::string& html) { + std::string url = "data:text/html;charset=utf-8," + html; + fill_data_.url = GURL(url); } void UpdateRendererIDs() { @@ -769,10 +769,10 @@ class PasswordAutofillAgentTest : public ChromeRenderViewTest { tracker->AjaxSucceeded(); } - void FireDidCommitProvisionalLoad() { + void FireDidFinishSameDocumentNavigation() { FormTracker* tracker = autofill_agent_->form_tracker_for_testing(); static_cast<content::RenderFrameObserver*>(tracker) - ->DidCommitProvisionalLoad(true, ui::PAGE_TRANSITION_LINK); + ->DidFinishSameDocumentNavigation(); } void ClearField(FormFieldData* field) { @@ -832,7 +832,7 @@ TEST_F(PasswordAutofillAgentTest, InitialAutocompleteForEmptyAction) { UpdateUsernameAndPasswordElements(); // Set the expected form origin and action URLs. - UpdateOriginForHTML(kEmptyActionFormHTML); + UpdateUrlForHTML(kEmptyActionFormHTML); fill_data_.action = GURL(); // Simulate the browser sending back the login info, it triggers the @@ -1327,7 +1327,7 @@ TEST_F(PasswordAutofillAgentTest, CreateScriptToRegisterListeners(kPasswordName, &password_event_checkers); std::string html = std::string(kFormHTML) + events_registration_script; LoadHTML(html.c_str()); - UpdateOriginForHTML(html); + UpdateUrlForHTML(html); UpdateUsernameAndPasswordElements(); // Simulate the browser sending back the login info, it triggers the @@ -1364,7 +1364,7 @@ TEST_F(PasswordAutofillAgentTest, CreateScriptToRegisterListeners(kPasswordName, &event_checkers); std::string html = std::string(kFormHTML) + events_registration_script; LoadHTML(html.c_str()); - UpdateOriginForHTML(html); + UpdateUrlForHTML(html); UpdateUsernameAndPasswordElements(); // Simulate the browser sending back the login info, it triggers the @@ -1834,7 +1834,7 @@ TEST_F(PasswordAutofillAgentTest, TouchToFillClosed) { // Reload the page and simulate fill. LoadHTML(kFormHTML); - UpdateOriginForHTML(kFormHTML); + UpdateUrlForHTML(kFormHTML); UpdateUsernameAndPasswordElements(); SimulateOnFillPasswordForm(fill_data_); @@ -1985,16 +1985,33 @@ TEST_F(PasswordAutofillAgentTest, ClickAndSelect) { CheckTextFieldsDOMState(kAliceUsername, true, kAlicePassword, true); } +TEST_F(PasswordAutofillAgentTest, + NoPopupOnPasswordFieldWithoutSuggestionsByDefault) { + ClearUsernameAndPasswordFields(); + UpdateRendererIDs(); + + // A call to InformNoSavedCredentials(should_show_popup_without_passwords) is + // what informs the agent whether it should show the popup even without + // suggestions. In this test, that call hasn't happened yet, so the popup + // should NOT show up without suggestions. + + SimulateElementClick(kPasswordName); + + EXPECT_CALL(fake_driver_, ShowPasswordSuggestions).Times(0); + base::RunLoop().RunUntilIdle(); +} // With butter, passwords fields should always trigger the popup so the user can // unlock account-stored suggestions from there. TEST_F(PasswordAutofillAgentTest, ShowPopupOnPasswordFieldWithoutSuggestions) { - base::test::ScopedFeatureList scoped_feature_list; - scoped_feature_list.InitAndEnableFeature( - password_manager::features::kEnablePasswordsAccountStorage); ClearUsernameAndPasswordFields(); UpdateRendererIDs(); + // InformNoSavedCredentials(should_show_popup_without_passwords) tells the + // agent to show the popup even without suggestions. + password_autofill_agent_->InformNoSavedCredentials( + /*should_show_popup_without_passwords=*/true); + SimulateElementClick(kPasswordName); EXPECT_CALL(fake_driver_, ShowPasswordSuggestions); @@ -2004,12 +2021,14 @@ TEST_F(PasswordAutofillAgentTest, ShowPopupOnPasswordFieldWithoutSuggestions) { // Before butter, passwords fields should never trigger the popup on password // passwords fields without suggestions since it would not be helpful. TEST_F(PasswordAutofillAgentTest, NoPopupOnPasswordFieldWithoutSuggestions) { - base::test::ScopedFeatureList scoped_feature_list; - scoped_feature_list.InitAndDisableFeature( - password_manager::features::kEnablePasswordsAccountStorage); ClearUsernameAndPasswordFields(); UpdateRendererIDs(); + // InformNoSavedCredentials(should_show_popup_without_passwords) tells the + // agent NOT to show the popup without suggestions. + password_autofill_agent_->InformNoSavedCredentials( + /*should_show_popup_without_passwords=*/false); + SimulateElementClick(kPasswordName); EXPECT_CALL(fake_driver_, ShowPasswordSuggestions).Times(0); @@ -2454,7 +2473,7 @@ TEST_F(PasswordAutofillAgentTest, ShowPopupOnEmptyPasswordField) { LoadHTML(kVisibleFormWithNoUsernameHTML); UpdateOnlyPasswordElement(); fill_data_.username_field = FormFieldData(); - UpdateOriginForHTML(kVisibleFormWithNoUsernameHTML); + UpdateUrlForHTML(kVisibleFormWithNoUsernameHTML); fill_data_.additional_logins.clear(); password_element_.SetValue(""); @@ -2479,7 +2498,7 @@ TEST_F(PasswordAutofillAgentTest, ShowPopupOnAutofilledPasswordField) { LoadHTML(kVisibleFormWithNoUsernameHTML); UpdateOnlyPasswordElement(); fill_data_.username_field = FormFieldData(); - UpdateOriginForHTML(kVisibleFormWithNoUsernameHTML); + UpdateUrlForHTML(kVisibleFormWithNoUsernameHTML); fill_data_.additional_logins.clear(); password_element_.SetValue(""); @@ -2504,7 +2523,7 @@ TEST_F(PasswordAutofillAgentTest, NotShowPopupPasswordField) { LoadHTML(kVisibleFormWithNoUsernameHTML); UpdateOnlyPasswordElement(); fill_data_.username_field = FormFieldData(); - UpdateOriginForHTML(kVisibleFormWithNoUsernameHTML); + UpdateUrlForHTML(kVisibleFormWithNoUsernameHTML); fill_data_.additional_logins.clear(); password_element_.SetValue(""); @@ -2669,7 +2688,7 @@ TEST_F(PasswordAutofillAgentTest, PasswordGenerationSupersedesAutofill) { fill_data_.wait_for_username = true; fill_data_.username_field = FormFieldData(); fill_data_.password_field.name = ASCIIToUTF16("new_password"); - UpdateOriginForHTML(kSignupFormHTML); + UpdateUrlForHTML(kSignupFormHTML); SimulateOnFillPasswordForm(fill_data_); // Simulate generation triggering. @@ -2697,7 +2716,7 @@ TEST_F(PasswordAutofillAgentTest, PasswordGenerationSupersedesAutofill) { // password. TEST_F(PasswordAutofillAgentTest, FillSuggestionPasswordChangeForms) { LoadHTML(kPasswordChangeFormHTML); - UpdateOriginForHTML(kPasswordChangeFormHTML); + UpdateUrlForHTML(kPasswordChangeFormHTML); UpdateUsernameAndPasswordElements(); // Simulate the browser sending the login info, but set |wait_for_username| // to prevent the form from being immediately filled. @@ -2722,7 +2741,7 @@ TEST_F(PasswordAutofillAgentTest, FillSuggestionPasswordChangeForms) { TEST_F(PasswordAutofillAgentTest, SuggestionsOnUsernameFieldOfChangePasswordForm) { LoadHTML(kPasswordChangeFormHTML); - UpdateOriginForHTML(kPasswordChangeFormHTML); + UpdateUrlForHTML(kPasswordChangeFormHTML); UpdateUsernameAndPasswordElements(); ClearUsernameAndPasswordFields(); @@ -2739,7 +2758,7 @@ TEST_F(PasswordAutofillAgentTest, TEST_F(PasswordAutofillAgentTest, SuggestionsOnPasswordFieldOfChangePasswordForm) { LoadHTML(kPasswordChangeFormHTML); - UpdateOriginForHTML(kPasswordChangeFormHTML); + UpdateUrlForHTML(kPasswordChangeFormHTML); UpdateUsernameAndPasswordElements(); ClearUsernameAndPasswordFields(); @@ -3193,7 +3212,7 @@ TEST_F(PasswordAutofillAgentTest, password_element_ = control_elements[0].To<WebInputElement>(); } - UpdateOriginForHTML(test_case.html_form); + UpdateUrlForHTML(test_case.html_form); if (test_case.does_trigger_autocomplete_on_fill) { // Prepare |fill_data_| to trigger autocomplete. fill_data_.username_field.name = @@ -3247,7 +3266,7 @@ TEST_F(PasswordAutofillAgentTest, ShowSuggestionForNonUsernameFieldForms) { LoadHTML(kTwoNoUsernameFormsHTML); fill_data_.username_field.name.clear(); fill_data_.username_field.value.clear(); - UpdateOriginForHTML(kTwoNoUsernameFormsHTML); + UpdateUrlForHTML(kTwoNoUsernameFormsHTML); SimulateOnFillPasswordForm(fill_data_); SimulateElementClick("password1"); @@ -3354,7 +3373,7 @@ TEST_F(PasswordAutofillAgentTest, SuggestPasswordFieldSignInForm) { // only on this field. TEST_F(PasswordAutofillAgentTest, SuggestMultiplePasswordFields) { LoadHTML(kPasswordChangeFormHTML); - UpdateOriginForHTML(kPasswordChangeFormHTML); + UpdateUrlForHTML(kPasswordChangeFormHTML); UpdateUsernameAndPasswordElements(); // Simulate the browser sending back the login info. @@ -3455,7 +3474,7 @@ TEST_F(PasswordAutofillAgentTest, "form.parentNode.removeChild(form);"; ExecuteJavaScriptForTests(remove_form.c_str()); - FireDidCommitProvisionalLoad(); + FireDidFinishSameDocumentNavigation(); ExpectSameDocumentNavigationWithUsernameAndPasswords( renderer_id, std::string(), "random", std::string(), @@ -3501,7 +3520,7 @@ TEST_F(PasswordAutofillAgentTest, // between discovering a form and receving credentials from the browser process. TEST_F(PasswordAutofillAgentTest, AutocompleteWhenPageUrlIsChanged) { // Simulate that JavaScript changes url. - fill_data_.origin = GURL(fill_data_.origin.possibly_invalid_spec() + "/path"); + fill_data_.url = GURL(fill_data_.url.possibly_invalid_spec() + "/path"); SimulateOnFillPasswordForm(fill_data_); @@ -3574,7 +3593,7 @@ TEST_F(PasswordAutofillAgentTest, ManualFallbackForSaving) { TEST_F(PasswordAutofillAgentTest, ManualFallbackForSaving_PasswordChangeForm) { LoadHTML(kPasswordChangeFormHTML); - UpdateOriginForHTML(kPasswordChangeFormHTML); + UpdateUrlForHTML(kPasswordChangeFormHTML); UpdateUsernameAndPasswordElements(); // No password to save yet - no fallback. @@ -3686,7 +3705,7 @@ TEST_F(PasswordAutofillAgentTest, PSLMatchedPasswordIsNotAutofill) { UpdateUsernameAndPasswordElements(); // Set the expected form origin and action URLs. - UpdateOriginForHTML(kFormWithPrefilledUsernameHTML); + UpdateUrlForHTML(kFormWithPrefilledUsernameHTML); // Add PSL matched credentials with username equal to prefilled one. PasswordAndMetadata psl_credentials; diff --git a/chromium/chrome/renderer/browser_exposed_renderer_interfaces.cc b/chromium/chrome/renderer/browser_exposed_renderer_interfaces.cc index 7e668b98964..02b8ac4e26a 100644 --- a/chromium/chrome/renderer/browser_exposed_renderer_interfaces.cc +++ b/chromium/chrome/renderer/browser_exposed_renderer_interfaces.cc @@ -12,27 +12,22 @@ #include "chrome/renderer/chrome_content_renderer_client.h" #include "chrome/renderer/chrome_render_thread_observer.h" #include "chrome/renderer/media/webrtc_logging_agent_impl.h" -#include "components/safe_browsing/buildflags.h" #include "components/spellcheck/spellcheck_buildflags.h" #include "components/visitedlink/renderer/visitedlink_reader.h" #include "components/web_cache/renderer/web_cache_impl.h" #include "mojo/public/cpp/bindings/binder_map.h" -#if BUILDFLAG(FULL_SAFE_BROWSING) -#include "chrome/renderer/safe_browsing/phishing_classifier_delegate.h" -#endif - #if BUILDFLAG(ENABLE_SPELLCHECK) #include "components/spellcheck/renderer/spellcheck.h" #endif -#if defined(OS_LINUX) +#if defined(OS_CHROMEOS) #include "base/allocator/buildflags.h" #if BUILDFLAG(USE_TCMALLOC) #include "chrome/common/performance_manager/mojom/tcmalloc.mojom.h" #include "chrome/renderer/performance_manager/mechanisms/tcmalloc_tunables_impl.h" #endif // BUILDFLAG(USE_TCMALLOC) -#endif // defined(OS_LINUX) +#endif // defined(OS_CHROMEOS) namespace { @@ -67,20 +62,14 @@ void ExposeChromeRendererInterfacesToBrowser( binders->Add(base::BindRepeating(&BindWebRTCLoggingAgent, client), base::SequencedTaskRunnerHandle::Get()); -#if BUILDFLAG(FULL_SAFE_BROWSING) - binders->Add( - base::BindRepeating(&safe_browsing::PhishingClassifierFilter::Create), - base::SequencedTaskRunnerHandle::Get()); -#endif - -#if defined(OS_LINUX) +#if defined(OS_CHROMEOS) #if BUILDFLAG(USE_TCMALLOC) binders->Add( base::BindRepeating( &performance_manager::mechanism::TcmallocTunablesImpl::Create), base::SequencedTaskRunnerHandle::Get()); #endif // BUILDFLAG(USE_TCMALLOC) -#endif // defined(OS_LINUX) +#endif // defined(OS_CHROMEOS) #if BUILDFLAG(ENABLE_SPELLCHECK) binders->Add(base::BindRepeating(&BindSpellChecker, client), diff --git a/chromium/chrome/renderer/chrome_content_renderer_client.cc b/chromium/chrome/renderer/chrome_content_renderer_client.cc index c1d92ae00fe..35542225d86 100644 --- a/chromium/chrome/renderer/chrome_content_renderer_client.cc +++ b/chromium/chrome/renderer/chrome_content_renderer_client.cc @@ -34,11 +34,11 @@ #include "chrome/common/crash_keys.h" #include "chrome/common/pdf_util.h" #include "chrome/common/pepper_permission_util.h" -#include "chrome/common/prerender_types.h" #include "chrome/common/prerender_url_loader_throttle.h" +#include "chrome/common/privacy_budget/privacy_budget_settings_provider.h" #include "chrome/common/profiler/thread_profiler.h" #include "chrome/common/render_messages.h" -#include "chrome/common/secure_origin_whitelist.h" +#include "chrome/common/secure_origin_allowlist.h" #include "chrome/common/url_constants.h" #include "chrome/grit/chromium_strings.h" #include "chrome/grit/generated_resources.h" @@ -48,6 +48,8 @@ #include "chrome/renderer/chrome_content_settings_agent_delegate.h" #include "chrome/renderer/chrome_render_frame_observer.h" #include "chrome/renderer/chrome_render_thread_observer.h" +#include "chrome/renderer/lite_video/lite_video_hint_agent.h" +#include "chrome/renderer/lite_video/lite_video_util.h" #include "chrome/renderer/loadtimes_extension_bindings.h" #include "chrome/renderer/media/flash_embed_rewrite.h" #include "chrome/renderer/media/webrtc_logging_agent_impl.h" @@ -83,10 +85,12 @@ #include "components/dom_distiller/core/url_constants.h" #include "components/error_page/common/error.h" #include "components/error_page/common/localized_error.h" +#include "components/grit/components_scaled_resources.h" #include "components/network_hints/renderer/web_prescient_networking_impl.h" #include "components/page_load_metrics/renderer/metrics_render_frame_observer.h" #include "components/paint_preview/buildflags/buildflags.h" #include "components/pdf/renderer/pepper_pdf_host.h" +#include "components/prerender/common/prerender_types.mojom.h" #include "components/safe_browsing/buildflags.h" #include "components/safe_browsing/content/renderer/threat_dom_details.h" #include "components/spellcheck/spellcheck_buildflags.h" @@ -126,6 +130,7 @@ #include "services/service_manager/public/cpp/interface_provider.h" #include "third_party/blink/public/common/associated_interfaces/associated_interface_provider.h" #include "third_party/blink/public/common/features.h" +#include "third_party/blink/public/common/privacy_budget/identifiability_study_settings.h" #include "third_party/blink/public/mojom/fetch/fetch_api_request.mojom-shared.h" #include "third_party/blink/public/mojom/page/page_visibility_state.mojom.h" #include "third_party/blink/public/platform/platform.h" @@ -338,6 +343,9 @@ ChromeContentRendererClient::ChromeContentRendererClient() for (const char* origin : kPredefinedAllowedCameraDeviceOrigins) allowed_camera_device_origins_.insert(origin); #endif + + blink::IdentifiabilityStudySettings::SetGlobalProvider( + std::make_unique<PrivacyBudgetSettingsProvider>()); } ChromeContentRendererClient::~ChromeContentRendererClient() {} @@ -443,7 +451,7 @@ void ChromeContentRendererClient::RenderThreadStarted() { } for (auto& scheme : - secure_origin_whitelist::GetSchemesBypassingSecureContextCheck()) { + secure_origin_allowlist::GetSchemesBypassingSecureContextCheck()) { WebSecurityPolicy::AddSchemeToSecureContextSafelist( WebString::FromASCII(scheme)); } @@ -546,9 +554,9 @@ void ChromeContentRendererClient::RenderFrameCreated( } } - // Set up a mojo service to test if this page is a distiller page. + // Set up a render frame observer to test if this page is a distiller page. new dom_distiller::DistillerJsRenderFrameObserver( - render_frame, ISOLATED_WORLD_ID_CHROME_INTERNAL, registry); + render_frame, ISOLATED_WORLD_ID_CHROME_INTERNAL); if (dom_distiller::ShouldStartDistillabilityService()) { // Create DistillabilityAgent to send distillability updates to @@ -601,10 +609,13 @@ void ChromeContentRendererClient::RenderFrameCreated( render_frame, subresource_filter_ruleset_dealer_.get(), std::move(ad_resource_tracker)); } - if (render_frame->IsMainFrame()) { - new previews::ResourceLoadingHintsAgent( - render_frame_observer->associated_interfaces(), render_frame); - } + + if (lite_video::IsLiteVideoEnabled()) + new lite_video::LiteVideoHintAgent(render_frame); + + new previews::ResourceLoadingHintsAgent( + render_frame_observer->associated_interfaces(), render_frame); + if (translate::IsSubFrameTranslationEnabled()) { new translate::PerFrameTranslateAgent( render_frame, ISOLATED_WORLD_ID_TRANSLATE, associated_interfaces); @@ -1332,7 +1343,7 @@ bool ChromeContentRendererClient::IsPrefetchOnly( content::RenderFrame* render_frame, const blink::WebURLRequest& request) { return prerender::PrerenderHelper::GetPrerenderMode(render_frame) == - prerender::PREFETCH_ONLY; + prerender::mojom::PrerenderMode::kPrefetchOnly; } uint64_t ChromeContentRendererClient::VisitedLinkHash(const char* canonical_url, @@ -1438,8 +1449,10 @@ ChromeContentRendererClient::CreateWorkerContentSettingsClient( #if !defined(OS_ANDROID) std::unique_ptr<media::SpeechRecognitionClient> ChromeContentRendererClient::CreateSpeechRecognitionClient( - content::RenderFrame* render_frame) { - return std::make_unique<ChromeSpeechRecognitionClient>(render_frame); + content::RenderFrame* render_frame, + media::SpeechRecognitionClient::OnReadyCallback callback) { + return std::make_unique<ChromeSpeechRecognitionClient>(render_frame, + std::move(callback)); } #endif diff --git a/chromium/chrome/renderer/chrome_content_renderer_client.h b/chromium/chrome/renderer/chrome_content_renderer_client.h index 416b7a8f3ba..ea214daadf1 100644 --- a/chromium/chrome/renderer/chrome_content_renderer_client.h +++ b/chromium/chrome/renderer/chrome_content_renderer_client.h @@ -151,7 +151,8 @@ class ChromeContentRendererClient content::RenderFrame* render_frame) override; #if !defined(OS_ANDROID) std::unique_ptr<media::SpeechRecognitionClient> CreateSpeechRecognitionClient( - content::RenderFrame* render_frame) override; + content::RenderFrame* render_frame, + media::SpeechRecognitionClient::OnReadyCallback callback) override; #endif void AddSupportedKeySystems( std::vector<std::unique_ptr<::media::KeySystemProperties>>* key_systems) diff --git a/chromium/chrome/renderer/chrome_content_renderer_client_browsertest.cc b/chromium/chrome/renderer/chrome_content_renderer_client_browsertest.cc index ae1c0079376..8df56778bfb 100644 --- a/chromium/chrome/renderer/chrome_content_renderer_client_browsertest.cc +++ b/chromium/chrome/renderer/chrome_content_renderer_client_browsertest.cc @@ -11,7 +11,6 @@ #include "base/command_line.h" #include "base/strings/stringprintf.h" #include "base/strings/utf_string_conversions.h" -#include "base/task/post_task.h" #include "base/threading/thread_task_runner_handle.h" #include "chrome/browser/ui/browser.h" #include "chrome/browser/ui/tabs/tab_strip_model.h" @@ -136,8 +135,8 @@ class ChromeContentRendererClientBrowserTest : EXPECT_EQ(request.relative_url, GetParam().expected_url) << "URL is wrong for test " << GetParam().name; - base::PostTask(FROM_HERE, {content::BrowserThread::UI}, - message_runner_->QuitClosure()); + content::GetUIThreadTaskRunner({})->PostTask( + FROM_HERE, message_runner_->QuitClosure()); } void WaitForYouTubeRequest() { diff --git a/chromium/chrome/renderer/chrome_content_renderer_client_unittest.cc b/chromium/chrome/renderer/chrome_content_renderer_client_unittest.cc index 3025593fd2a..7974a07b096 100644 --- a/chromium/chrome/renderer/chrome_content_renderer_client_unittest.cc +++ b/chromium/chrome/renderer/chrome_content_renderer_client_unittest.cc @@ -16,12 +16,14 @@ #include "base/strings/utf_string_conversions.h" #include "base/values.h" #include "build/build_config.h" +#include "chrome/common/privacy_budget/scoped_privacy_budget_config.h" #include "chrome/renderer/searchbox/search_bouncer.h" #include "components/data_reduction_proxy/core/common/data_reduction_proxy_headers.h" #include "content/public/common/content_switches.h" #include "content/public/common/webplugininfo.h" #include "extensions/buildflags/buildflags.h" #include "testing/gtest/include/gtest/gtest.h" +#include "third_party/blink/public/common/privacy_budget/identifiability_study_settings.h" #include "third_party/blink/public/platform/web_string.h" #include "third_party/blink/public/platform/web_url_response.h" #include "url/gurl.h" @@ -232,6 +234,8 @@ TEST_F(ChromeContentRendererClientTest, NaClRestriction) { // SearchBouncer doesn't exist on Android. #if !defined(OS_ANDROID) TEST_F(ChromeContentRendererClientTest, ShouldSuppressErrorPage) { + test::ScopedPrivacyBudgetConfig config( + test::ScopedPrivacyBudgetConfig::kDisable); ChromeContentRendererClient client; SearchBouncer::GetInstance()->SetNewTabPageURL(GURL("http://example.com/n")); EXPECT_FALSE(client.ShouldSuppressErrorPage( @@ -242,6 +246,8 @@ TEST_F(ChromeContentRendererClientTest, ShouldSuppressErrorPage) { } TEST_F(ChromeContentRendererClientTest, ShouldTrackUseCounter) { + test::ScopedPrivacyBudgetConfig config( + test::ScopedPrivacyBudgetConfig::kDisable); ChromeContentRendererClient client; SearchBouncer::GetInstance()->SetNewTabPageURL(GURL("http://example.com/n")); EXPECT_TRUE(client.ShouldTrackUseCounter(GURL("http://example.com"))); @@ -249,3 +255,19 @@ TEST_F(ChromeContentRendererClientTest, ShouldTrackUseCounter) { SearchBouncer::GetInstance()->SetNewTabPageURL(GURL::EmptyGURL()); } #endif + +TEST_F(ChromeContentRendererClientTest, IdentifiabilityStudySettingsDisabled) { + test::ScopedPrivacyBudgetConfig::Parameters parameters; + parameters.enabled = false; + test::ScopedPrivacyBudgetConfig config(parameters); + ChromeContentRendererClient client; + EXPECT_FALSE(blink::IdentifiabilityStudySettings::Get()->IsActive()); +} + +TEST_F(ChromeContentRendererClientTest, IdentifiabilityStudySettingsEnabled) { + test::ScopedPrivacyBudgetConfig::Parameters parameters; + parameters.enabled = true; + test::ScopedPrivacyBudgetConfig config(parameters); + ChromeContentRendererClient client; + EXPECT_TRUE(blink::IdentifiabilityStudySettings::Get()->IsActive()); +} diff --git a/chromium/chrome/renderer/chrome_content_settings_agent_delegate.cc b/chromium/chrome/renderer/chrome_content_settings_agent_delegate.cc index 77187dca7f9..0c4dcfb794b 100644 --- a/chromium/chrome/renderer/chrome_content_settings_agent_delegate.cc +++ b/chromium/chrome/renderer/chrome_content_settings_agent_delegate.cc @@ -152,9 +152,8 @@ bool ChromeContentSettingsAgentDelegate::OnMessageReceived( } void ChromeContentSettingsAgentDelegate::DidCommitProvisionalLoad( - bool is_same_document_navigation, ui::PageTransition transition) { - if (render_frame()->GetWebFrame()->Parent() || is_same_document_navigation) + if (render_frame()->GetWebFrame()->Parent()) return; temporarily_allowed_plugins_.clear(); diff --git a/chromium/chrome/renderer/chrome_content_settings_agent_delegate.h b/chromium/chrome/renderer/chrome_content_settings_agent_delegate.h index cfc027255b3..b56cc2cd8ef 100644 --- a/chromium/chrome/renderer/chrome_content_settings_agent_delegate.h +++ b/chromium/chrome/renderer/chrome_content_settings_agent_delegate.h @@ -49,8 +49,7 @@ class ChromeContentSettingsAgentDelegate // RenderFrameObserver: bool OnMessageReceived(const IPC::Message& message) override; - void DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) override; + void DidCommitProvisionalLoad(ui::PageTransition transition) override; void OnDestruct() override; void OnLoadBlockedPlugins(const std::string& identifier); diff --git a/chromium/chrome/renderer/chrome_render_frame_observer.cc b/chromium/chrome/renderer/chrome_render_frame_observer.cc index ac6ba14b9a1..a545474a8a0 100644 --- a/chromium/chrome/renderer/chrome_render_frame_observer.cc +++ b/chromium/chrome/renderer/chrome_render_frame_observer.cc @@ -22,13 +22,13 @@ #include "chrome/common/chrome_isolated_world_ids.h" #include "chrome/common/chrome_switches.h" #include "chrome/common/open_search_description_document_handler.mojom.h" -#include "chrome/common/prerender_messages.h" #include "chrome/common/render_messages.h" #include "chrome/renderer/media/media_feeds.h" #include "chrome/renderer/prerender/prerender_helper.h" #include "chrome/renderer/web_apps.h" #include "components/crash/core/common/crash_key.h" #include "components/offline_pages/buildflags/buildflags.h" +#include "components/prerender/common/prerender_messages.h" #include "components/translate/content/renderer/translate_agent.h" #include "components/translate/core/common/translate_util.h" #include "components/web_cache/renderer/web_cache_impl.h" @@ -130,11 +130,11 @@ ChromeRenderFrameObserver::ChromeRenderFrameObserver( if (!render_frame->IsMainFrame()) return; -#if BUILDFLAG(SAFE_BROWSING_CSD) +#if BUILDFLAG(FULL_SAFE_BROWSING) const base::CommandLine& command_line = *base::CommandLine::ForCurrentProcess(); if (!command_line.HasSwitch(switches::kDisableClientSidePhishingDetection)) - SetClientSidePhishingDetection(true); + SetClientSidePhishingDetection(); #endif if (!translate::IsSubFrameTranslationEnabled()) { translate_agent_ = @@ -193,10 +193,123 @@ bool ChromeRenderFrameObserver::OnMessageReceived(const IPC::Message& message) { return handled; } +void ChromeRenderFrameObserver::ReadyToCommitNavigation( + WebDocumentLoader* document_loader) { + // Execute cache clear operations that were postponed until a navigation + // event (including tab reload). + if (render_frame()->IsMainFrame() && web_cache_impl_) + web_cache_impl_->ExecutePendingClearCache(); + + // Let translate_agent do any preparatory work for loading a URL. + if (!translate_agent_) + return; + + translate_agent_->PrepareForUrl( + render_frame()->GetWebFrame()->GetDocument().Url()); +} + +void ChromeRenderFrameObserver::DidFinishLoad() { + WebLocalFrame* frame = render_frame()->GetWebFrame(); + // Don't do anything for subframes. + if (frame->Parent()) + return; + + GURL osdd_url = frame->GetDocument().OpenSearchDescriptionURL(); + if (!osdd_url.is_empty()) { + mojo::AssociatedRemote<chrome::mojom::OpenSearchDescriptionDocumentHandler> + osdd_handler; + render_frame()->GetRemoteAssociatedInterfaces()->GetInterface( + &osdd_handler); + osdd_handler->PageHasOpenSearchDescriptionDocument( + frame->GetDocument().Url(), osdd_url); + } +} + +void ChromeRenderFrameObserver::DidCreateNewDocument() { +#if BUILDFLAG(ENABLE_OFFLINE_PAGES) + DCHECK(render_frame()); + if (!render_frame()->IsMainFrame()) + return; + + DCHECK(render_frame()->GetWebFrame()); + blink::WebDocumentLoader* doc_loader = + render_frame()->GetWebFrame()->GetDocumentLoader(); + DCHECK(doc_loader); + + if (!doc_loader->HasBeenLoadedAsWebArchive()) + return; + + // Connect to Mojo service on browser to notify it of the page's archive + // properties. + mojo::AssociatedRemote<offline_pages::mojom::MhtmlPageNotifier> + mhtml_notifier; + render_frame()->GetRemoteAssociatedInterfaces()->GetInterface( + &mhtml_notifier); + DCHECK(mhtml_notifier); + blink::WebArchiveInfo info = doc_loader->GetArchiveInfo(); + + mhtml_notifier->NotifyMhtmlPageLoadAttempted(info.load_result, info.url, + info.date); +#endif +} + +void ChromeRenderFrameObserver::DidCommitProvisionalLoad( + ui::PageTransition transition) { + WebLocalFrame* frame = render_frame()->GetWebFrame(); + + // Don't do anything for subframes. + if (frame->Parent()) + return; + + static crash_reporter::CrashKeyString<8> view_count_key("view-count"); + view_count_key.Set( + base::NumberToString(content::RenderView::GetRenderViewCount())); + +#if !defined(OS_ANDROID) + if (render_frame()->GetEnabledBindings() & + content::kWebUIBindingsPolicyMask) { + for (const auto& script : webui_javascript_) + render_frame()->ExecuteJavaScript(script); + webui_javascript_.clear(); + } +#endif +} + +void ChromeRenderFrameObserver::DidClearWindowObject() { +#if !defined(OS_ANDROID) + const base::CommandLine& command_line = + *base::CommandLine::ForCurrentProcess(); + if (command_line.HasSwitch(switches::kInstantProcess)) + SearchBoxExtension::Install(render_frame()->GetWebFrame()); +#endif // !defined(OS_ANDROID) +} + +void ChromeRenderFrameObserver::DidMeaningfulLayout( + blink::WebMeaningfulLayout layout_type) { + // Don't do any work for subframes. + if (!render_frame()->IsMainFrame()) + return; + + switch (layout_type) { + case blink::WebMeaningfulLayout::kFinishedParsing: + CapturePageText(PRELIMINARY_CAPTURE); + break; + case blink::WebMeaningfulLayout::kFinishedLoading: + CapturePageText(FINAL_CAPTURE); + break; + default: + break; + } +} + +void ChromeRenderFrameObserver::OnDestruct() { + delete this; +} + void ChromeRenderFrameObserver::OnSetIsPrerendering( - prerender::PrerenderMode mode, + prerender::mojom::PrerenderMode mode, const std::string& histogram_prefix) { - if (mode != prerender::NO_PRERENDER) { + if (mode != prerender::mojom::PrerenderMode::kNoPrerender) { // If the PrerenderHelper for this frame already exists, don't create it. It // can already be created for subframes during handling of // RenderFrameCreated, if the parent frame was prerendering at time of @@ -210,14 +323,17 @@ void ChromeRenderFrameObserver::OnSetIsPrerendering( } } -void ChromeRenderFrameObserver::RequestReloadImageForContextNode() { - WebLocalFrame* frame = render_frame()->GetWebFrame(); - // TODO(dglazkov): This code is clearly in the wrong place. Need - // to investigate what it is doing and fix (http://crbug.com/606164). - WebNode context_node = frame->ContextMenuNode(); - if (!context_node.IsNull()) { - frame->ReloadImage(context_node); - } +void ChromeRenderFrameObserver::SetWindowFeatures( + blink::mojom::WindowFeaturesPtr window_features) { + render_frame()->GetRenderView()->GetWebView()->SetWindowFeatures( + content::ConvertMojoWindowFeaturesToWebWindowFeatures(*window_features)); +} + +void ChromeRenderFrameObserver::ExecuteWebUIJavaScript( + const base::string16& javascript) { +#if !defined(OS_ANDROID) + webui_javascript_.push_back(javascript); +#endif } void ChromeRenderFrameObserver::RequestImageForContextNode( @@ -294,6 +410,16 @@ void ChromeRenderFrameObserver::RequestImageForContextNode( std::move(callback).Run(image_data, original_size, image_extension); } +void ChromeRenderFrameObserver::RequestReloadImageForContextNode() { + WebLocalFrame* frame = render_frame()->GetWebFrame(); + // TODO(dglazkov): This code is clearly in the wrong place. Need + // to investigate what it is doing and fix (http://crbug.com/606164). + WebNode context_node = frame->ContextMenuNode(); + if (!context_node.IsNull()) { + frame->ReloadImage(context_node); + } +} + void ChromeRenderFrameObserver::GetWebApplicationInfo( GetWebApplicationInfoCallback callback) { WebLocalFrame* frame = render_frame()->GetWebFrame(); @@ -345,114 +471,17 @@ void ChromeRenderFrameObserver::GetMediaFeedURL( std::move(callback).Run(MediaFeeds::GetMediaFeedURL(render_frame())); } -void ChromeRenderFrameObserver::SetClientSidePhishingDetection( - bool enable_phishing_detection) { -#if BUILDFLAG(SAFE_BROWSING_CSD) - phishing_classifier_ = - enable_phishing_detection - ? safe_browsing::PhishingClassifierDelegate::Create(render_frame(), - nullptr) - : nullptr; -#endif -} - -void ChromeRenderFrameObserver::ExecuteWebUIJavaScript( - const base::string16& javascript) { -#if !defined(OS_ANDROID) - webui_javascript_.push_back(javascript); -#endif -} - -void ChromeRenderFrameObserver::DidFinishLoad() { - WebLocalFrame* frame = render_frame()->GetWebFrame(); - // Don't do anything for subframes. - if (frame->Parent()) - return; - - GURL osdd_url = frame->GetDocument().OpenSearchDescriptionURL(); - if (!osdd_url.is_empty()) { - mojo::AssociatedRemote<chrome::mojom::OpenSearchDescriptionDocumentHandler> - osdd_handler; - render_frame()->GetRemoteAssociatedInterfaces()->GetInterface( - &osdd_handler); - osdd_handler->PageHasOpenSearchDescriptionDocument( - frame->GetDocument().Url(), osdd_url); - } -} - -void ChromeRenderFrameObserver::DidCreateNewDocument() { -#if BUILDFLAG(ENABLE_OFFLINE_PAGES) - DCHECK(render_frame()); - if (!render_frame()->IsMainFrame()) - return; - - DCHECK(render_frame()->GetWebFrame()); - blink::WebDocumentLoader* doc_loader = - render_frame()->GetWebFrame()->GetDocumentLoader(); - DCHECK(doc_loader); - - if (!doc_loader->HasBeenLoadedAsWebArchive()) - return; - - // Connect to Mojo service on browser to notify it of the page's archive - // properties. - mojo::AssociatedRemote<offline_pages::mojom::MhtmlPageNotifier> - mhtml_notifier; - render_frame()->GetRemoteAssociatedInterfaces()->GetInterface( - &mhtml_notifier); - DCHECK(mhtml_notifier); - blink::WebArchiveInfo info = doc_loader->GetArchiveInfo(); - - mhtml_notifier->NotifyMhtmlPageLoadAttempted(info.load_result, info.url, - info.date); -#endif -} - -void ChromeRenderFrameObserver::ReadyToCommitNavigation( - WebDocumentLoader* document_loader) { - // Execute cache clear operations that were postponed until a navigation - // event (including tab reload). - if (render_frame()->IsMainFrame() && web_cache_impl_) - web_cache_impl_->ExecutePendingClearCache(); - - // Let translate_agent do any preparatory work for loading a URL. - if (!translate_agent_) - return; - - translate_agent_->PrepareForUrl( - render_frame()->GetWebFrame()->GetDocument().Url()); -} - -void ChromeRenderFrameObserver::DidCommitProvisionalLoad( - bool is_same_document_navigation, - ui::PageTransition transition) { - WebLocalFrame* frame = render_frame()->GetWebFrame(); - - // Don't do anything for subframes. - if (frame->Parent()) - return; - - static crash_reporter::CrashKeyString<8> view_count_key("view-count"); - view_count_key.Set( - base::NumberToString(content::RenderView::GetRenderViewCount())); - -#if !defined(OS_ANDROID) - if (render_frame()->GetEnabledBindings() & - content::kWebUIBindingsPolicyMask) { - for (const auto& script : webui_javascript_) - render_frame()->ExecuteJavaScript(script); - webui_javascript_.clear(); - } +void ChromeRenderFrameObserver::SetClientSidePhishingDetection() { +#if BUILDFLAG(FULL_SAFE_BROWSING) + phishing_classifier_ = safe_browsing::PhishingClassifierDelegate::Create( + render_frame(), nullptr); #endif } -void ChromeRenderFrameObserver::DidClearWindowObject() { -#if !defined(OS_ANDROID) - const base::CommandLine& command_line = - *base::CommandLine::ForCurrentProcess(); - if (command_line.HasSwitch(switches::kInstantProcess)) - SearchBoxExtension::Install(render_frame()->GetWebFrame()); -#endif // !defined(OS_ANDROID) +void ChromeRenderFrameObserver::OnRenderFrameObserverRequest( + mojo::PendingAssociatedReceiver<chrome::mojom::ChromeRenderFrame> + receiver) { + receivers_.Add(this, std::move(receiver)); } void ChromeRenderFrameObserver::CapturePageText(TextCaptureType capture_type) { @@ -479,7 +508,7 @@ void ChromeRenderFrameObserver::CapturePageText(TextCaptureType capture_type) { // Don't capture contents unless there is either a translate agent or a // phishing classifier to consume them. -#if BUILDFLAG(SAFE_BROWSING_CSD) +#if BUILDFLAG(FULL_SAFE_BROWSING) if (!translate_agent_ && !phishing_classifier_) return; #else @@ -509,7 +538,7 @@ void ChromeRenderFrameObserver::CapturePageText(TextCaptureType capture_type) { TRACE_EVENT0("renderer", "ChromeRenderFrameObserver::CapturePageText"); -#if BUILDFLAG(SAFE_BROWSING_CSD) +#if BUILDFLAG(FULL_SAFE_BROWSING) // Will swap out the string. if (phishing_classifier_) phishing_classifier_->PageCaptured(&contents, @@ -517,39 +546,6 @@ void ChromeRenderFrameObserver::CapturePageText(TextCaptureType capture_type) { #endif } -void ChromeRenderFrameObserver::DidMeaningfulLayout( - blink::WebMeaningfulLayout layout_type) { - // Don't do any work for subframes. - if (!render_frame()->IsMainFrame()) - return; - - switch (layout_type) { - case blink::WebMeaningfulLayout::kFinishedParsing: - CapturePageText(PRELIMINARY_CAPTURE); - break; - case blink::WebMeaningfulLayout::kFinishedLoading: - CapturePageText(FINAL_CAPTURE); - break; - default: - break; - } -} - -void ChromeRenderFrameObserver::OnDestruct() { - delete this; -} - -void ChromeRenderFrameObserver::OnRenderFrameObserverRequest( - mojo::PendingAssociatedReceiver<chrome::mojom::ChromeRenderFrame> - receiver) { - receivers_.Add(this, std::move(receiver)); -} - -void ChromeRenderFrameObserver::SetWindowFeatures( - blink::mojom::WindowFeaturesPtr window_features) { - render_frame()->GetRenderView()->GetWebView()->SetWindowFeatures( - content::ConvertMojoWindowFeaturesToWebWindowFeatures(*window_features)); -} // static bool ChromeRenderFrameObserver::NeedsDownscale( diff --git a/chromium/chrome/renderer/chrome_render_frame_observer.h b/chromium/chrome/renderer/chrome_render_frame_observer.h index a7257c75d0f..2680019005e 100644 --- a/chromium/chrome/renderer/chrome_render_frame_observer.h +++ b/chromium/chrome/renderer/chrome_render_frame_observer.h @@ -12,7 +12,7 @@ #include "base/timer/timer.h" #include "build/build_config.h" #include "chrome/common/chrome_render_frame.mojom.h" -#include "chrome/common/prerender_types.h" +#include "components/prerender/common/prerender_types.mojom.h" #include "components/safe_browsing/buildflags.h" #include "content/public/renderer/render_frame_observer.h" #include "mojo/public/cpp/bindings/associated_receiver_set.h" @@ -73,14 +73,13 @@ class ChromeRenderFrameObserver : public content::RenderFrameObserver, blink::WebDocumentLoader* document_loader) override; void DidFinishLoad() override; void DidCreateNewDocument() override; - void DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) override; + void DidCommitProvisionalLoad(ui::PageTransition transition) override; void DidClearWindowObject() override; void DidMeaningfulLayout(blink::WebMeaningfulLayout layout_type) override; void OnDestruct() override; // IPC handlers - void OnSetIsPrerendering(prerender::PrerenderMode mode, + void OnSetIsPrerendering(prerender::mojom::PrerenderMode mode, const std::string& histogram_prefix); // chrome::mojom::ChromeRenderFrame: @@ -93,13 +92,15 @@ class ChromeRenderFrameObserver : public content::RenderFrameObserver, chrome::mojom::ImageFormat image_format, RequestImageForContextNodeCallback callback) override; void RequestReloadImageForContextNode() override; - void SetClientSidePhishingDetection(bool enable_phishing_detection) override; void GetWebApplicationInfo(GetWebApplicationInfoCallback callback) override; #if defined(OS_ANDROID) void SetCCTClientHeader(const std::string& header) override; #endif void GetMediaFeedURL(GetMediaFeedURLCallback callback) override; + // Initialize a |phishing_classifier_delegate_|. + void SetClientSidePhishingDetection(); + void OnRenderFrameObserverRequest( mojo::PendingAssociatedReceiver<chrome::mojom::ChromeRenderFrame> receiver); @@ -130,7 +131,7 @@ class ChromeRenderFrameObserver : public content::RenderFrameObserver, // Have the same lifetime as us. translate::TranslateAgent* translate_agent_; -#if BUILDFLAG(SAFE_BROWSING_CSD) +#if BUILDFLAG(FULL_SAFE_BROWSING) safe_browsing::PhishingClassifierDelegate* phishing_classifier_ = nullptr; #endif @@ -150,4 +151,4 @@ class ChromeRenderFrameObserver : public content::RenderFrameObserver, DISALLOW_COPY_AND_ASSIGN(ChromeRenderFrameObserver); }; -#endif // CHROME_RENDERER_CHROME_RENDER_FRAME_OBSERVER_H_
\ No newline at end of file +#endif // CHROME_RENDERER_CHROME_RENDER_FRAME_OBSERVER_H_ diff --git a/chromium/chrome/renderer/extensions/cast_streaming_native_handler.cc b/chromium/chrome/renderer/extensions/cast_streaming_native_handler.cc deleted file mode 100644 index 1fb0aa4b760..00000000000 --- a/chromium/chrome/renderer/extensions/cast_streaming_native_handler.cc +++ /dev/null @@ -1,844 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/extensions/cast_streaming_native_handler.h" - -#include <stddef.h> -#include <stdint.h> - -#include <algorithm> -#include <functional> -#include <iterator> -#include <memory> -#include <string> -#include <utility> -#include <vector> - -#include "base/bind.h" -#include "base/location.h" -#include "base/logging.h" -#include "base/memory/scoped_refptr.h" -#include "base/single_thread_task_runner.h" -#include "base/stl_util.h" -#include "base/strings/string_number_conversions.h" -#include "base/system/sys_info.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/values.h" -#include "chrome/common/extensions/api/cast_streaming_receiver_session.h" -#include "chrome/common/extensions/api/cast_streaming_rtp_stream.h" -#include "chrome/common/extensions/api/cast_streaming_udp_transport.h" -#include "chrome/renderer/media/cast_receiver_session.h" -#include "chrome/renderer/media/cast_rtp_stream.h" -#include "chrome/renderer/media/cast_session.h" -#include "chrome/renderer/media/cast_udp_transport.h" -#include "content/public/renderer/v8_value_converter.h" -#include "extensions/common/extension.h" -#include "extensions/renderer/native_extension_bindings_system.h" -#include "extensions/renderer/script_context.h" -#include "media/base/audio_parameters.h" -#include "media/base/limits.h" -#include "net/base/host_port_pair.h" -#include "net/base/ip_address.h" -#include "third_party/blink/public/platform/web_media_stream.h" -#include "third_party/blink/public/platform/web_media_stream_track.h" -#include "third_party/blink/public/platform/web_url.h" -#include "third_party/blink/public/web/web_dom_media_stream_track.h" -#include "third_party/blink/public/web/web_local_frame.h" -#include "url/gurl.h" - -using content::V8ValueConverter; -using media::cast::FrameSenderConfig; - -// Extension types. -using extensions::api::cast_streaming_receiver_session::RtpReceiverParams; -using extensions::api::cast_streaming_rtp_stream::RtpParams; -using extensions::api::cast_streaming_rtp_stream::RtpPayloadParams; -using extensions::api::cast_streaming_udp_transport::IPEndPoint; - -namespace extensions { - -namespace { - -constexpr char kInvalidAesIvMask[] = "Invalid value for AES IV mask"; -constexpr char kInvalidAesKey[] = "Invalid value for AES key"; -constexpr char kInvalidDestination[] = "Invalid destination"; -constexpr char kInvalidRtpParams[] = "Invalid value for RTP params"; -constexpr char kInvalidStreamArgs[] = "Invalid stream arguments"; -constexpr char kRtpStreamNotFound[] = "The RTP stream cannot be found"; -constexpr char kUdpTransportNotFound[] = "The UDP transport cannot be found"; -constexpr char kUnableToConvertArgs[] = "Unable to convert arguments"; -constexpr char kUnableToConvertParams[] = "Unable to convert params"; -constexpr char kCodecNameOpus[] = "OPUS"; -constexpr char kCodecNameVp8[] = "VP8"; -constexpr char kCodecNameH264[] = "H264"; -constexpr char kCodecNameRemoteAudio[] = "REMOTE_AUDIO"; -constexpr char kCodecNameRemoteVideo[] = "REMOTE_VIDEO"; - -// To convert from kilobits per second to bits per second. -constexpr int kBitsPerKilobit = 1000; - -bool HexDecode(const std::string& input, std::string* output) { - DCHECK(output->empty()); - if (!base::HexStringToString(input, output)) { - output->clear(); - return false; - } - return true; -} - -int NumberOfEncodeThreads() { - // Do not saturate CPU utilization just for encoding. On a lower-end system - // with only 1 or 2 cores, use only one thread for encoding. On systems with - // more cores, allow half of the cores to be used for encoding. - return std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2); -} - -bool ToFrameSenderConfigOrThrow(v8::Isolate* isolate, - const RtpPayloadParams& ext_params, - FrameSenderConfig* config) { - config->sender_ssrc = ext_params.ssrc; - config->receiver_ssrc = ext_params.feedback_ssrc; - if (config->sender_ssrc == config->receiver_ssrc) { - DVLOG(1) << "sender_ssrc " << config->sender_ssrc - << " cannot be equal to receiver_ssrc"; - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - config->min_playout_delay = base::TimeDelta::FromMilliseconds( - ext_params.min_latency ? *ext_params.min_latency - : ext_params.max_latency); - config->max_playout_delay = - base::TimeDelta::FromMilliseconds(ext_params.max_latency); - config->animated_playout_delay = base::TimeDelta::FromMilliseconds( - ext_params.animated_latency ? *ext_params.animated_latency - : ext_params.max_latency); - if (config->min_playout_delay <= base::TimeDelta()) { - DVLOG(1) << "min_playout_delay " << config->min_playout_delay - << " must be greater than zero"; - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - if (config->min_playout_delay > config->max_playout_delay) { - DVLOG(1) << "min_playout_delay " << config->min_playout_delay - << " must be less than or equal to max_palyout_delay"; - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - if (config->animated_playout_delay < config->min_playout_delay || - config->animated_playout_delay > config->max_playout_delay) { - DVLOG(1) << "animated_playout_delay " << config->animated_playout_delay - << " must be between (inclusive) the min and max playout delay"; - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - if (ext_params.codec_name == kCodecNameOpus) { - config->rtp_payload_type = media::cast::RtpPayloadType::AUDIO_OPUS; - config->use_external_encoder = false; - config->rtp_timebase = ext_params.clock_rate - ? *ext_params.clock_rate - : media::cast::kDefaultAudioSamplingRate; - // Sampling rate must be one of the Opus-supported values. - switch (config->rtp_timebase) { - case 48000: - case 24000: - case 16000: - case 12000: - case 8000: - break; - default: - DVLOG(1) << "rtp_timebase " << config->rtp_timebase << " is invalid"; - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - config->channels = ext_params.channels ? *ext_params.channels : 2; - if (config->channels != 1 && config->channels != 2) { - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - DVLOG(1) << "channels " << config->channels << " is invalid"; - return false; - } - config->min_bitrate = config->start_bitrate = config->max_bitrate = - ext_params.max_bitrate ? (*ext_params.max_bitrate) * kBitsPerKilobit - : media::cast::kDefaultAudioEncoderBitrate; - config->max_frame_rate = 100; // 10ms audio frames. - config->codec = media::cast::CODEC_AUDIO_OPUS; - } else if (ext_params.codec_name == kCodecNameVp8 || - ext_params.codec_name == kCodecNameH264) { - config->rtp_timebase = media::cast::kVideoFrequency; - config->channels = ext_params.channels ? *ext_params.channels : 1; - if (config->channels != 1) { - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - DVLOG(1) << "channels " << config->channels << " is invalid"; - return false; - } - config->min_bitrate = ext_params.min_bitrate - ? (*ext_params.min_bitrate) * kBitsPerKilobit - : media::cast::kDefaultMinVideoBitrate; - config->max_bitrate = ext_params.max_bitrate - ? (*ext_params.max_bitrate) * kBitsPerKilobit - : media::cast::kDefaultMaxVideoBitrate; - if (config->min_bitrate > config->max_bitrate) { - DVLOG(1) << "min_bitrate " << config->min_bitrate << " is larger than " - << "max_bitrate " << config->max_bitrate; - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - config->start_bitrate = config->min_bitrate; - config->max_frame_rate = std::max( - 1.0, ext_params.max_frame_rate ? *ext_params.max_frame_rate : 0.0); - if (config->max_frame_rate > media::limits::kMaxFramesPerSecond) { - DVLOG(1) << "max_frame_rate " << config->max_frame_rate << " is invalid"; - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - if (ext_params.codec_name == kCodecNameVp8) { - config->rtp_payload_type = media::cast::RtpPayloadType::VIDEO_VP8; - config->codec = media::cast::CODEC_VIDEO_VP8; - config->use_external_encoder = - CastRtpStream::IsHardwareVP8EncodingSupported(); - } else { - config->rtp_payload_type = media::cast::RtpPayloadType::VIDEO_H264; - config->codec = media::cast::CODEC_VIDEO_H264; - config->use_external_encoder = - CastRtpStream::IsHardwareH264EncodingSupported(); - } - if (!config->use_external_encoder) - config->video_codec_params.number_of_encode_threads = - NumberOfEncodeThreads(); - } else if (ext_params.codec_name == kCodecNameRemoteAudio) { - config->rtp_payload_type = media::cast::RtpPayloadType::REMOTE_AUDIO; - config->codec = media::cast::CODEC_AUDIO_REMOTE; - } else if (ext_params.codec_name == kCodecNameRemoteVideo) { - config->rtp_payload_type = media::cast::RtpPayloadType::REMOTE_VIDEO; - config->codec = media::cast::CODEC_VIDEO_REMOTE; - } else { - DVLOG(1) << "codec_name " << ext_params.codec_name << " is invalid"; - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - if (ext_params.aes_key && !HexDecode(*ext_params.aes_key, &config->aes_key)) { - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidAesKey, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - if (ext_params.aes_iv_mask && - !HexDecode(*ext_params.aes_iv_mask, &config->aes_iv_mask)) { - isolate->ThrowException( - v8::Exception::Error(v8::String::NewFromUtf8(isolate, kInvalidAesIvMask, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - return true; -} - -void FromFrameSenderConfig(const FrameSenderConfig& config, - RtpPayloadParams* ext_params) { - ext_params->payload_type = static_cast<int>(config.rtp_payload_type); - ext_params->max_latency = config.max_playout_delay.InMilliseconds(); - ext_params->min_latency.reset( - new int(config.min_playout_delay.InMilliseconds())); - ext_params->animated_latency.reset( - new int(config.animated_playout_delay.InMilliseconds())); - switch (config.codec) { - case media::cast::CODEC_AUDIO_OPUS: - ext_params->codec_name = kCodecNameOpus; - break; - case media::cast::CODEC_VIDEO_VP8: - ext_params->codec_name = kCodecNameVp8; - break; - case media::cast::CODEC_VIDEO_H264: - ext_params->codec_name = kCodecNameH264; - break; - case media::cast::CODEC_AUDIO_REMOTE: - ext_params->codec_name = kCodecNameRemoteAudio; - break; - case media::cast::CODEC_VIDEO_REMOTE: - ext_params->codec_name = kCodecNameRemoteVideo; - break; - default: - NOTREACHED(); - } - ext_params->ssrc = config.sender_ssrc; - ext_params->feedback_ssrc = config.receiver_ssrc; - if (config.rtp_timebase) - ext_params->clock_rate.reset(new int(config.rtp_timebase)); - if (config.min_bitrate) - ext_params->min_bitrate.reset( - new int(config.min_bitrate / kBitsPerKilobit)); - if (config.max_bitrate) - ext_params->max_bitrate.reset( - new int(config.max_bitrate / kBitsPerKilobit)); - if (config.channels) - ext_params->channels.reset(new int(config.channels)); - if (config.max_frame_rate > 0.0) - ext_params->max_frame_rate.reset(new double(config.max_frame_rate)); -} - -} // namespace - -// |last_transport_id_| is the identifier for the next created RTP stream. To -// create globally unique IDs used for referring to RTP stream objects in -// browser process, we set its higher 16 bits as HASH(extension_id)&0x7fff, and -// lower 16 bits as the sequence number of the RTP stream created in the same -// extension. Collision will happen when the first RTP stream keeps alive after -// creating another 64k-1 RTP streams in the same extension, which is very -// unlikely to happen in normal use cases. -CastStreamingNativeHandler::CastStreamingNativeHandler( - ScriptContext* context, - NativeExtensionBindingsSystem* bindings_system) - : ObjectBackedNativeHandler(context), - last_transport_id_( - context->extension() - ? (((base::Hash(context->extension()->id()) & 0x7fff) << 16) + 1) - : 1), - bindings_system_(bindings_system) {} - -CastStreamingNativeHandler::~CastStreamingNativeHandler() { - // Note: A superclass's destructor will call Invalidate(), but Invalidate() - // may also be called at any time before destruction. -} - -void CastStreamingNativeHandler::AddRoutes() { - RouteHandlerFunction( - "CreateSession", "cast.streaming.session", - base::BindRepeating(&CastStreamingNativeHandler::CreateCastSession, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "DestroyCastRtpStream", "cast.streaming.rtpStream", - base::BindRepeating(&CastStreamingNativeHandler::DestroyCastRtpStream, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "GetSupportedParamsCastRtpStream", "cast.streaming.rtpStream", - base::BindRepeating( - &CastStreamingNativeHandler::GetSupportedParamsCastRtpStream, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "StartCastRtpStream", "cast.streaming.rtpStream", - base::BindRepeating(&CastStreamingNativeHandler::StartCastRtpStream, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "StopCastRtpStream", "cast.streaming.rtpStream", - base::BindRepeating(&CastStreamingNativeHandler::StopCastRtpStream, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "DestroyCastUdpTransport", "cast.streaming.udpTransport", - base::BindRepeating(&CastStreamingNativeHandler::DestroyCastUdpTransport, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "SetDestinationCastUdpTransport", "cast.streaming.udpTransport", - base::BindRepeating( - &CastStreamingNativeHandler::SetDestinationCastUdpTransport, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "SetOptionsCastUdpTransport", "cast.streaming.udpTransport", - base::BindRepeating( - &CastStreamingNativeHandler::SetOptionsCastUdpTransport, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "ToggleLogging", "cast.streaming.rtpStream", - base::BindRepeating(&CastStreamingNativeHandler::ToggleLogging, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "GetRawEvents", "cast.streaming.rtpStream", - base::BindRepeating(&CastStreamingNativeHandler::GetRawEvents, - weak_factory_.GetWeakPtr())); - RouteHandlerFunction( - "GetStats", "cast.streaming.rtpStream", - base::BindRepeating(&CastStreamingNativeHandler::GetStats, - weak_factory_.GetWeakPtr())); -} - -void CastStreamingNativeHandler::Invalidate() { - // Cancel all function call routing and callbacks. - weak_factory_.InvalidateWeakPtrs(); - - // Clear all references to V8 and Cast objects, which will trigger - // auto-destructions (effectively stopping all sessions). - get_stats_callbacks_.clear(); - get_raw_events_callbacks_.clear(); - create_callback_.Reset(); - udp_transport_map_.clear(); - rtp_stream_map_.clear(); - - ObjectBackedNativeHandler::Invalidate(); -} - -void CastStreamingNativeHandler::CreateCastSession( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(3, args.Length()); - CHECK(args[2]->IsFunction()); - - v8::Isolate* isolate = context()->v8_context()->GetIsolate(); - - auto session = base::MakeRefCounted<CastSession>( - context()->web_frame()->GetTaskRunner(blink::TaskType::kInternalMedia)); - std::unique_ptr<CastRtpStream> stream1, stream2; - if ((args[0]->IsNull() || args[0]->IsUndefined()) && - (args[1]->IsNull() || args[1]->IsUndefined())) { - DVLOG(3) << "CreateCastSession for remoting."; - // Creates audio/video RTP streams for media remoting. - stream1.reset(new CastRtpStream(true, session)); - stream2.reset(new CastRtpStream(false, session)); - } else { - // Creates RTP streams that consume from an audio and/or a video - // MediaStreamTrack. - if (!args[0]->IsNull() && !args[0]->IsUndefined()) { - CHECK(args[0]->IsObject()); - blink::WebDOMMediaStreamTrack track = - blink::WebDOMMediaStreamTrack::FromV8Value(args[0]); - if (track.IsNull()) { - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidStreamArgs, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return; - } - stream1.reset(new CastRtpStream(track.Component(), session)); - } - if (!args[1]->IsNull() && !args[1]->IsUndefined()) { - CHECK(args[1]->IsObject()); - blink::WebDOMMediaStreamTrack track = - blink::WebDOMMediaStreamTrack::FromV8Value(args[1]); - if (track.IsNull()) { - isolate->ThrowException(v8::Exception::Error( - v8::String::NewFromUtf8(isolate, kInvalidStreamArgs, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return; - } - stream2.reset(new CastRtpStream(track.Component(), session)); - } - } - std::unique_ptr<CastUdpTransport> udp_transport( - new CastUdpTransport(session)); - - create_callback_.Reset(isolate, args[2].As<v8::Function>()); - - context() - ->web_frame() - ->GetTaskRunner(blink::TaskType::kInternalMedia) - ->PostTask(FROM_HERE, - base::BindOnce(&CastStreamingNativeHandler::CallCreateCallback, - weak_factory_.GetWeakPtr(), std::move(stream1), - std::move(stream2), std::move(udp_transport))); -} - -void CastStreamingNativeHandler::CallCreateCallback( - std::unique_ptr<CastRtpStream> stream1, - std::unique_ptr<CastRtpStream> stream2, - std::unique_ptr<CastUdpTransport> udp_transport) { - v8::Isolate* isolate = context()->isolate(); - v8::HandleScope handle_scope(isolate); - v8::Context::Scope context_scope(context()->v8_context()); - - v8::Local<v8::Value> callback_args[3]; - callback_args[0] = v8::Null(isolate); - callback_args[1] = v8::Null(isolate); - - if (stream1) { - const int stream1_id = last_transport_id_++; - callback_args[0] = v8::Integer::New(isolate, stream1_id); - rtp_stream_map_[stream1_id] = std::move(stream1); - } - if (stream2) { - const int stream2_id = last_transport_id_++; - callback_args[1] = v8::Integer::New(isolate, stream2_id); - rtp_stream_map_[stream2_id] = std::move(stream2); - } - const int udp_id = last_transport_id_++; - udp_transport_map_[udp_id] = std::move(udp_transport); - callback_args[2] = v8::Integer::New(isolate, udp_id); - context()->SafeCallFunction( - v8::Local<v8::Function>::New(isolate, create_callback_), 3, - callback_args); - create_callback_.Reset(); -} - -void CastStreamingNativeHandler::CallStartCallback(int stream_id) const { - base::ListValue event_args; - event_args.AppendInteger(stream_id); - bindings_system_->DispatchEventInContext("cast.streaming.rtpStream.onStarted", - &event_args, nullptr, context()); -} - -void CastStreamingNativeHandler::CallStopCallback(int stream_id) const { - base::ListValue event_args; - event_args.AppendInteger(stream_id); - bindings_system_->DispatchEventInContext("cast.streaming.rtpStream.onStopped", - &event_args, nullptr, context()); -} - -void CastStreamingNativeHandler::CallErrorCallback( - int stream_id, - const std::string& message) const { - base::ListValue event_args; - event_args.AppendInteger(stream_id); - event_args.AppendString(message); - bindings_system_->DispatchEventInContext("cast.streaming.rtpStream.onError", - &event_args, nullptr, context()); -} - -void CastStreamingNativeHandler::DestroyCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(1, args.Length()); - CHECK(args[0]->IsInt32()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - if (!GetRtpStreamOrThrow(transport_id)) - return; - rtp_stream_map_.erase(transport_id); -} - -void CastStreamingNativeHandler::GetSupportedParamsCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args) const { - CHECK_EQ(1, args.Length()); - CHECK(args[0]->IsInt32()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - CastRtpStream* transport = GetRtpStreamOrThrow(transport_id); - if (!transport) - return; - - std::unique_ptr<V8ValueConverter> converter = V8ValueConverter::Create(); - std::vector<FrameSenderConfig> configs = transport->GetSupportedConfigs(); - v8::Local<v8::Array> result = - v8::Array::New(args.GetIsolate(), static_cast<int>(configs.size())); - for (size_t i = 0; i < configs.size(); ++i) { - RtpParams params; - FromFrameSenderConfig(configs[i], ¶ms.payload); - std::unique_ptr<base::DictionaryValue> params_value = params.ToValue(); - result - ->CreateDataProperty( - context()->v8_context(), static_cast<int>(i), - converter->ToV8Value(params_value.get(), context()->v8_context())) - .Check(); - } - args.GetReturnValue().Set(result); -} - -void CastStreamingNativeHandler::StartCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(2, args.Length()); - CHECK(args[0]->IsInt32()); - CHECK(args[1]->IsObject()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - CastRtpStream* transport = GetRtpStreamOrThrow(transport_id); - if (!transport) - return; - - std::unique_ptr<base::Value> params_value = - V8ValueConverter::Create()->FromV8Value(args[1], context()->v8_context()); - if (!params_value) { - args.GetIsolate()->ThrowException(v8::Exception::TypeError( - v8::String::NewFromUtf8(args.GetIsolate(), kUnableToConvertParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return; - } - std::unique_ptr<RtpParams> params = RtpParams::FromValue(*params_value); - if (!params) { - args.GetIsolate()->ThrowException(v8::Exception::TypeError( - v8::String::NewFromUtf8(args.GetIsolate(), kInvalidRtpParams, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return; - } - - FrameSenderConfig config; - v8::Isolate* isolate = context()->v8_context()->GetIsolate(); - if (!ToFrameSenderConfigOrThrow(isolate, params->payload, &config)) - return; - - base::OnceClosure start_callback = - base::BindOnce(&CastStreamingNativeHandler::CallStartCallback, - weak_factory_.GetWeakPtr(), transport_id); - base::OnceClosure stop_callback = - base::BindOnce(&CastStreamingNativeHandler::CallStopCallback, - weak_factory_.GetWeakPtr(), transport_id); - CastRtpStream::ErrorCallback error_callback = - base::BindRepeating(&CastStreamingNativeHandler::CallErrorCallback, - weak_factory_.GetWeakPtr(), transport_id); - - // |transport_id| is a globally unique identifier for the RTP stream. - transport->Start(transport_id, config, std::move(start_callback), - std::move(stop_callback), error_callback); -} - -void CastStreamingNativeHandler::StopCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(1, args.Length()); - CHECK(args[0]->IsInt32()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - CastRtpStream* transport = GetRtpStreamOrThrow(transport_id); - if (!transport) - return; - transport->Stop(); -} - -void CastStreamingNativeHandler::DestroyCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(1, args.Length()); - CHECK(args[0]->IsInt32()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - if (!GetUdpTransportOrThrow(transport_id)) - return; - udp_transport_map_.erase(transport_id); -} - -void CastStreamingNativeHandler::SetDestinationCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(2, args.Length()); - CHECK(args[0]->IsInt32()); - CHECK(args[1]->IsObject()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - CastUdpTransport* transport = GetUdpTransportOrThrow(transport_id); - if (!transport) - return; - - net::IPEndPoint dest; - if (!IPEndPointFromArg(args.GetIsolate(), - args[1], - &dest)) { - return; - } - transport->SetDestination( - dest, - base::Bind(&CastStreamingNativeHandler::CallErrorCallback, - weak_factory_.GetWeakPtr(), - transport_id)); -} - -void CastStreamingNativeHandler::SetOptionsCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(2, args.Length()); - CHECK(args[0]->IsInt32()); - CHECK(args[1]->IsObject()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - CastUdpTransport* transport = GetUdpTransportOrThrow(transport_id); - if (!transport) - return; - - std::unique_ptr<base::DictionaryValue> options = - base::DictionaryValue::From(V8ValueConverter::Create()->FromV8Value( - args[1], context()->v8_context())); - if (!options) { - args.GetIsolate()->ThrowException(v8::Exception::TypeError( - v8::String::NewFromUtf8(args.GetIsolate(), kUnableToConvertArgs, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return; - } - transport->SetOptions(std::move(options)); -} - -void CastStreamingNativeHandler::ToggleLogging( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(2, args.Length()); - CHECK(args[0]->IsInt32()); - CHECK(args[1]->IsBoolean()); - - const int stream_id = args[0].As<v8::Int32>()->Value(); - CastRtpStream* stream = GetRtpStreamOrThrow(stream_id); - if (!stream) - return; - - const bool enable = args[1]->ToBoolean(args.GetIsolate())->Value(); - stream->ToggleLogging(enable); -} - -void CastStreamingNativeHandler::GetRawEvents( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(3, args.Length()); - CHECK(args[0]->IsInt32()); - CHECK(args[1]->IsNull() || args[1]->IsString()); - CHECK(args[2]->IsFunction()); - - const int transport_id = args[0].As<v8::Int32>()->Value(); - v8::Isolate* isolate = args.GetIsolate(); - v8::Global<v8::Function> callback(isolate, args[2].As<v8::Function>()); - std::string extra_data; - if (!args[1]->IsNull()) { - extra_data = *v8::String::Utf8Value(isolate, args[1]); - } - - CastRtpStream* transport = GetRtpStreamOrThrow(transport_id); - if (!transport) - return; - - get_raw_events_callbacks_.insert( - std::make_pair(transport_id, std::move(callback))); - - transport->GetRawEvents( - base::Bind(&CastStreamingNativeHandler::CallGetRawEventsCallback, - weak_factory_.GetWeakPtr(), - transport_id), - extra_data); -} - -void CastStreamingNativeHandler::GetStats( - const v8::FunctionCallbackInfo<v8::Value>& args) { - CHECK_EQ(2, args.Length()); - CHECK(args[0]->IsInt32()); - CHECK(args[1]->IsFunction()); - const int transport_id = args[0].As<v8::Int32>()->Value(); - CastRtpStream* transport = GetRtpStreamOrThrow(transport_id); - if (!transport) - return; - - v8::Global<v8::Function> callback(args.GetIsolate(), - args[1].As<v8::Function>()); - get_stats_callbacks_.insert( - std::make_pair(transport_id, std::move(callback))); - - transport->GetStats( - base::Bind(&CastStreamingNativeHandler::CallGetStatsCallback, - weak_factory_.GetWeakPtr(), - transport_id)); -} - -void CastStreamingNativeHandler::CallGetRawEventsCallback( - int transport_id, - std::unique_ptr<base::Value> raw_events) { - v8::Isolate* isolate = context()->isolate(); - v8::HandleScope handle_scope(isolate); - v8::Context::Scope context_scope(context()->v8_context()); - - auto it = get_raw_events_callbacks_.find(transport_id); - if (it == get_raw_events_callbacks_.end()) - return; - v8::Local<v8::Value> callback_args[] = {V8ValueConverter::Create()->ToV8Value( - raw_events.get(), context()->v8_context())}; - context()->SafeCallFunction(v8::Local<v8::Function>::New(isolate, it->second), - base::size(callback_args), callback_args); - get_raw_events_callbacks_.erase(it); -} - -void CastStreamingNativeHandler::CallGetStatsCallback( - int transport_id, - std::unique_ptr<base::DictionaryValue> stats) { - v8::Isolate* isolate = context()->isolate(); - v8::HandleScope handle_scope(isolate); - v8::Context::Scope context_scope(context()->v8_context()); - - auto it = get_stats_callbacks_.find(transport_id); - if (it == get_stats_callbacks_.end()) - return; - - v8::Local<v8::Value> callback_args[] = {V8ValueConverter::Create()->ToV8Value( - stats.get(), context()->v8_context())}; - context()->SafeCallFunction(v8::Local<v8::Function>::New(isolate, it->second), - base::size(callback_args), callback_args); - get_stats_callbacks_.erase(it); -} - -CastRtpStream* CastStreamingNativeHandler::GetRtpStreamOrThrow( - int transport_id) const { - auto iter = rtp_stream_map_.find(transport_id); - if (iter != rtp_stream_map_.end()) - return iter->second.get(); - v8::Isolate* isolate = context()->v8_context()->GetIsolate(); - isolate->ThrowException(v8::Exception::RangeError( - v8::String::NewFromUtf8(isolate, kRtpStreamNotFound, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return NULL; -} - -CastUdpTransport* CastStreamingNativeHandler::GetUdpTransportOrThrow( - int transport_id) const { - auto iter = udp_transport_map_.find(transport_id); - if (iter != udp_transport_map_.end()) - return iter->second.get(); - v8::Isolate* isolate = context()->v8_context()->GetIsolate(); - isolate->ThrowException(v8::Exception::RangeError( - v8::String::NewFromUtf8(isolate, kUdpTransportNotFound, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return NULL; -} - -bool CastStreamingNativeHandler::IPEndPointFromArg( - v8::Isolate* isolate, - const v8::Local<v8::Value>& arg, - net::IPEndPoint* ip_endpoint) const { - std::unique_ptr<base::Value> destination_value = - V8ValueConverter::Create()->FromV8Value(arg, context()->v8_context()); - if (!destination_value) { - isolate->ThrowException(v8::Exception::TypeError( - v8::String::NewFromUtf8(isolate, kInvalidAesIvMask, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - std::unique_ptr<IPEndPoint> destination = - IPEndPoint::FromValue(*destination_value); - if (!destination) { - isolate->ThrowException(v8::Exception::TypeError( - v8::String::NewFromUtf8(isolate, kInvalidDestination, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - net::IPAddress ip; - if (!ip.AssignFromIPLiteral(destination->address)) { - isolate->ThrowException(v8::Exception::TypeError( - v8::String::NewFromUtf8(isolate, kInvalidDestination, - v8::NewStringType::kNormal) - .ToLocalChecked())); - return false; - } - *ip_endpoint = net::IPEndPoint(ip, destination->port); - return true; -} - -void CastStreamingNativeHandler::CallReceiverErrorCallback( - v8::CopyablePersistentTraits<v8::Function>::CopyablePersistent function, - const std::string& error_message) { - v8::Isolate* isolate = context()->v8_context()->GetIsolate(); - v8::Local<v8::Value> arg = - v8::String::NewFromUtf8(isolate, error_message.data(), - v8::NewStringType::kNormal, error_message.size()) - .ToLocalChecked(); - context()->SafeCallFunction(v8::Local<v8::Function>::New(isolate, function), - 1, &arg); -} - -} // namespace extensions diff --git a/chromium/chrome/renderer/extensions/cast_streaming_native_handler.h b/chromium/chrome/renderer/extensions/cast_streaming_native_handler.h deleted file mode 100644 index 9f135b35be2..00000000000 --- a/chromium/chrome/renderer/extensions/cast_streaming_native_handler.h +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_EXTENSIONS_CAST_STREAMING_NATIVE_HANDLER_H_ -#define CHROME_RENDERER_EXTENSIONS_CAST_STREAMING_NATIVE_HANDLER_H_ - -#include <map> -#include <memory> - -#include "base/macros.h" -#include "base/memory/weak_ptr.h" -#include "extensions/renderer/object_backed_native_handler.h" -#include "v8/include/v8.h" - -class CastRtpStream; -class CastUdpTransport; - -namespace base { -class DictionaryValue; -class Value; -} - -namespace net { -class IPEndPoint; -} - -namespace extensions { -class NativeExtensionBindingsSystem; - -// Native code that handle chrome.webrtc custom bindings. -class CastStreamingNativeHandler : public ObjectBackedNativeHandler { - public: - CastStreamingNativeHandler(ScriptContext* context, - NativeExtensionBindingsSystem* bindings_system); - ~CastStreamingNativeHandler() override; - - // ObjectBackedNativeHandler: - void AddRoutes() override; - - protected: - // Shut down all sessions and cancel any in-progress operations because the - // ScriptContext is about to become invalid. - void Invalidate() override; - - private: - void CreateCastSession( - const v8::FunctionCallbackInfo<v8::Value>& args); - - void DestroyCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args); - void CreateParamsCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args); - void GetSupportedParamsCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args) const; - void StartCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args); - void StopCastRtpStream( - const v8::FunctionCallbackInfo<v8::Value>& args); - - void DestroyCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args); - void SetDestinationCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args); - void SetOptionsCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args); - void StopCastUdpTransport( - const v8::FunctionCallbackInfo<v8::Value>& args); - - void ToggleLogging(const v8::FunctionCallbackInfo<v8::Value>& args); - void GetRawEvents(const v8::FunctionCallbackInfo<v8::Value>& args); - void GetStats(const v8::FunctionCallbackInfo<v8::Value>& args); - - // Helper method to call the v8 callback function after a session is - // created. - void CallCreateCallback(std::unique_ptr<CastRtpStream> stream1, - std::unique_ptr<CastRtpStream> stream2, - std::unique_ptr<CastUdpTransport> udp_transport); - - void CallStartCallback(int stream_id) const; - void CallStopCallback(int stream_id) const; - void CallErrorCallback(int stream_id, const std::string& message) const; - - // |function| is a javascript function that will take |error_message| as - // an argument. Called when something goes wrong in a cast receiver. - void CallReceiverErrorCallback( - v8::CopyablePersistentTraits<v8::Function>::CopyablePersistent function, - const std::string& error_message); - - void CallGetRawEventsCallback(int transport_id, - std::unique_ptr<base::Value> raw_events); - void CallGetStatsCallback(int transport_id, - std::unique_ptr<base::DictionaryValue> stats); - - // Gets the RTP stream or UDP transport indexed by an ID. - // If not found, returns NULL and throws a V8 exception. - CastRtpStream* GetRtpStreamOrThrow(int stream_id) const; - CastUdpTransport* GetUdpTransportOrThrow(int transport_id) const; - - bool IPEndPointFromArg(v8::Isolate* isolate, - const v8::Local<v8::Value>& arg, - net::IPEndPoint* ip_endpoint) const; - - int last_transport_id_; - - using RtpStreamMap = std::map<int, std::unique_ptr<CastRtpStream>>; - RtpStreamMap rtp_stream_map_; - - using UdpTransportMap = std::map<int, std::unique_ptr<CastUdpTransport>>; - UdpTransportMap udp_transport_map_; - - v8::Global<v8::Function> create_callback_; - - using RtpStreamCallbackMap = std::map<int, v8::Global<v8::Function>>; - RtpStreamCallbackMap get_raw_events_callbacks_; - RtpStreamCallbackMap get_stats_callbacks_; - - NativeExtensionBindingsSystem* bindings_system_; - - base::WeakPtrFactory<CastStreamingNativeHandler> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastStreamingNativeHandler); -}; - -} // namespace extensions - -#endif // CHROME_RENDERER_EXTENSIONS_CAST_STREAMING_NATIVE_HANDLER_H_ diff --git a/chromium/chrome/renderer/extensions/chrome_extensions_dispatcher_delegate.cc b/chromium/chrome/renderer/extensions/chrome_extensions_dispatcher_delegate.cc index 6621279f9f5..07605bbb35d 100644 --- a/chromium/chrome/renderer/extensions/chrome_extensions_dispatcher_delegate.cc +++ b/chromium/chrome/renderer/extensions/chrome_extensions_dispatcher_delegate.cc @@ -14,7 +14,6 @@ #include "chrome/grit/renderer_resources.h" #include "chrome/renderer/extensions/accessibility_private_hooks_delegate.h" #include "chrome/renderer/extensions/app_hooks_delegate.h" -#include "chrome/renderer/extensions/cast_streaming_native_handler.h" #include "chrome/renderer/extensions/extension_hooks_delegate.h" #include "chrome/renderer/extensions/media_galleries_custom_bindings.h" #include "chrome/renderer/extensions/notifications_native_handler.h" @@ -92,10 +91,6 @@ void ChromeExtensionsDispatcherDelegate::RegisterNativeHandlers( module_system->RegisterNativeHandler( "page_capture", std::unique_ptr<NativeHandler>( new extensions::PageCaptureCustomBindings(context))); - module_system->RegisterNativeHandler( - "cast_streaming_natives", - std::make_unique<extensions::CastStreamingNativeHandler>( - context, bindings_system)); // The following are native handlers that are defined in //extensions, but // are only used for APIs defined in Chrome. @@ -163,7 +158,7 @@ void ChromeExtensionsDispatcherDelegate::PopulateSourceMap( IDR_FILE_SYSTEM_PROVIDER_CUSTOM_BINDINGS_JS); source_map->RegisterSource("platformKeys", IDR_PLATFORM_KEYS_CUSTOM_BINDINGS_JS); - source_map->RegisterSource("platformKeys.getPublicKey", + source_map->RegisterSource("platformKeys.getPublicKeyUtil", IDR_PLATFORM_KEYS_GET_PUBLIC_KEY_JS); source_map->RegisterSource("platformKeys.internalAPI", IDR_PLATFORM_KEYS_INTERNAL_API_JS); @@ -179,18 +174,12 @@ void ChromeExtensionsDispatcherDelegate::PopulateSourceMap( IDR_IME_SERVICE_MOJOM_JS); source_map->RegisterSource("chromeos.ime.service", IDR_IME_SERVICE_BINDINGS_JS); + + source_map->RegisterSource("chromeos.tts.mojom.tts_stream.mojom", + IDR_TTS_STREAM_MOJOM_JS); + source_map->RegisterSource("chromeos.tts.stream", IDR_TTS_STREAM_BINDINGS_JS); #endif // defined(OS_CHROMEOS) - source_map->RegisterSource("cast.streaming.rtpStream", - IDR_CAST_STREAMING_RTP_STREAM_CUSTOM_BINDINGS_JS); - source_map->RegisterSource("cast.streaming.session", - IDR_CAST_STREAMING_SESSION_CUSTOM_BINDINGS_JS); - source_map->RegisterSource( - "cast.streaming.udpTransport", - IDR_CAST_STREAMING_UDP_TRANSPORT_CUSTOM_BINDINGS_JS); - source_map->RegisterSource( - "cast.streaming.receiverSession", - IDR_CAST_STREAMING_RECEIVER_SESSION_CUSTOM_BINDINGS_JS); source_map->RegisterSource( "webrtcDesktopCapturePrivate", IDR_WEBRTC_DESKTOP_CAPTURE_PRIVATE_CUSTOM_BINDINGS_JS); diff --git a/chromium/chrome/renderer/extensions/chrome_extensions_renderer_client.cc b/chromium/chrome/renderer/extensions/chrome_extensions_renderer_client.cc index c29b1f5a41f..6804fa38d51 100644 --- a/chromium/chrome/renderer/extensions/chrome_extensions_renderer_client.cc +++ b/chromium/chrome/renderer/extensions/chrome_extensions_renderer_client.cc @@ -21,7 +21,6 @@ #include "chrome/renderer/extensions/extension_process_policy.h" #include "chrome/renderer/extensions/renderer_permissions_policy_delegate.h" #include "chrome/renderer/extensions/resource_request_policy.h" -#include "chrome/renderer/media/cast_ipc_dispatcher.h" #include "content/public/common/content_constants.h" #include "content/public/common/content_switches.h" #include "content/public/renderer/render_frame.h" @@ -166,7 +165,6 @@ void ChromeExtensionsRendererClient::RenderThreadStarted() { thread->AddObserver(extension_dispatcher_.get()); thread->AddObserver(guest_view_container_dispatcher_.get()); - thread->AddFilter(new CastIPCDispatcher(thread->GetIOTaskRunner())); } void ChromeExtensionsRendererClient::RenderFrameCreated( diff --git a/chromium/chrome/renderer/extensions/platform_keys_natives.cc b/chromium/chrome/renderer/extensions/platform_keys_natives.cc index 831e007ccbe..22551ad0bca 100644 --- a/chromium/chrome/renderer/extensions/platform_keys_natives.cc +++ b/chromium/chrome/renderer/extensions/platform_keys_natives.cc @@ -10,8 +10,8 @@ #include "base/bind.h" #include "base/values.h" -#include "content/public/renderer/v8_value_converter.h" #include "extensions/renderer/script_context.h" +#include "gin/data_object_builder.h" #include "third_party/blink/public/platform/web_crypto_algorithm.h" #include "third_party/blink/public/platform/web_crypto_algorithm_params.h" #include "third_party/blink/public/platform/web_string.h" @@ -43,14 +43,17 @@ bool StringToWebCryptoOperation(const std::string& str, return false; } -std::unique_ptr<base::DictionaryValue> WebCryptoAlgorithmToBaseValue( - const blink::WebCryptoAlgorithm& algorithm) { +v8::Local<v8::Object> WebCryptoAlgorithmToV8Value( + const blink::WebCryptoAlgorithm& algorithm, + v8::Local<v8::Context> context) { DCHECK(!algorithm.IsNull()); + v8::Context::Scope scope(context); + v8::Isolate* isolate = context->GetIsolate(); - std::unique_ptr<base::DictionaryValue> dict(new base::DictionaryValue); const blink::WebCryptoAlgorithmInfo* info = blink::WebCryptoAlgorithm::LookupAlgorithmInfo(algorithm.Id()); - dict->SetKey("name", base::Value(info->name)); + gin::DataObjectBuilder builder(isolate); + builder.Set("name", base::StringPiece(info->name)); const blink::WebCryptoAlgorithm* hash = nullptr; @@ -59,16 +62,15 @@ std::unique_ptr<base::DictionaryValue> WebCryptoAlgorithmToBaseValue( const blink::WebCryptoRsaHashedKeyGenParams* rsa_hashed_key_gen = algorithm.RsaHashedKeyGenParams(); if (rsa_hashed_key_gen) { - dict->SetKey("modulusLength", - base::Value(static_cast<int>( - rsa_hashed_key_gen->ModulusLengthBits()))); + builder.Set("modulusLength", rsa_hashed_key_gen->ModulusLengthBits()); + const blink::WebVector<unsigned char>& public_exponent = rsa_hashed_key_gen->PublicExponent(); - dict->SetWithoutPathExpansion( - "publicExponent", - base::Value::CreateWithCopiedBuffer( - reinterpret_cast<const char*>(public_exponent.Data()), - public_exponent.size())); + v8::Local<v8::ArrayBuffer> buffer = + v8::ArrayBuffer::New(isolate, public_exponent.size()); + memcpy(buffer->GetContents().Data(), public_exponent.Data(), + public_exponent.size()); + builder.Set("publicExponent", buffer); hash = &rsa_hashed_key_gen->GetHash(); DCHECK(!hash->IsNull()); @@ -85,7 +87,7 @@ std::unique_ptr<base::DictionaryValue> WebCryptoAlgorithmToBaseValue( const blink::WebCryptoEcKeyGenParams* ec_key_gen = algorithm.EcKeyGenParams(); if (ec_key_gen) { - std::string named_curve; + base::StringPiece named_curve; switch (ec_key_gen->NamedCurve()) { case blink::kWebCryptoNamedCurveP256: named_curve = "P-256"; @@ -98,7 +100,7 @@ std::unique_ptr<base::DictionaryValue> WebCryptoAlgorithmToBaseValue( break; } DCHECK(!named_curve.empty()); - dict->SetKey("namedCurve", base::Value(std::move(named_curve))); + builder.Set("namedCurve", named_curve); } const blink::WebCryptoEcdsaParams* ecdsa = algorithm.EcdsaParams(); @@ -117,13 +119,13 @@ std::unique_ptr<base::DictionaryValue> WebCryptoAlgorithmToBaseValue( const blink::WebCryptoAlgorithmInfo* hash_info = blink::WebCryptoAlgorithm::LookupAlgorithmInfo(hash->Id()); - std::unique_ptr<base::DictionaryValue> hash_dict(new base::DictionaryValue); - hash_dict->SetKey("name", base::Value(hash_info->name)); - dict->SetWithoutPathExpansion("hash", std::move(hash_dict)); + builder.Set("hash", gin::DataObjectBuilder(isolate) + .Set("name", base::StringPiece(hash_info->name)) + .Build()); } // Otherwise, |algorithm| is missing support here or no parameters were // required. - return dict; + return builder.Build(); } } // namespace @@ -158,15 +160,11 @@ void PlatformKeysNatives::NormalizeAlgorithm( v8::Local<v8::Object>::Cast(call_info[0]), operation, &exception_code, &error_details, call_info.GetIsolate()); - std::unique_ptr<base::DictionaryValue> algorithm_dict; - if (!algorithm.IsNull()) - algorithm_dict = WebCryptoAlgorithmToBaseValue(algorithm); - - if (!algorithm_dict) + if (algorithm.IsNull()) return; - call_info.GetReturnValue().Set(content::V8ValueConverter::Create()->ToV8Value( - algorithm_dict.get(), context()->v8_context())); + call_info.GetReturnValue().Set( + WebCryptoAlgorithmToV8Value(algorithm, context()->v8_context())); } } // namespace extensions diff --git a/chromium/chrome/renderer/instant_restricted_id_cache.h b/chromium/chrome/renderer/instant_restricted_id_cache.h index 3062768bc22..6e3f820cf96 100644 --- a/chromium/chrome/renderer/instant_restricted_id_cache.h +++ b/chromium/chrome/renderer/instant_restricted_id_cache.h @@ -11,9 +11,9 @@ #include <utility> #include <vector> +#include "base/check_op.h" #include "base/containers/mru_cache.h" #include "base/gtest_prod_util.h" -#include "base/logging.h" #include "base/macros.h" #include "chrome/common/search/instant_types.h" diff --git a/chromium/chrome/renderer/lite_video/DEPS b/chromium/chrome/renderer/lite_video/DEPS new file mode 100644 index 00000000000..6511c3d7aac --- /dev/null +++ b/chromium/chrome/renderer/lite_video/DEPS @@ -0,0 +1,3 @@ +include_rules = [ +"+services/network/test", +] diff --git a/chromium/chrome/renderer/lite_video/OWNERS b/chromium/chrome/renderer/lite_video/OWNERS new file mode 100644 index 00000000000..d5e2286a680 --- /dev/null +++ b/chromium/chrome/renderer/lite_video/OWNERS @@ -0,0 +1,3 @@ +file://components/data_reduction_proxy/OWNERS + +# COMPONENT: Internal>Network>DataUse diff --git a/chromium/chrome/renderer/lite_video/lite_video_hint_agent.cc b/chromium/chrome/renderer/lite_video/lite_video_hint_agent.cc new file mode 100644 index 00000000000..4f4840c349e --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_hint_agent.cc @@ -0,0 +1,99 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/renderer/lite_video/lite_video_hint_agent.h" + +#include "base/metrics/histogram_macros.h" +#include "chrome/renderer/lite_video/lite_video_util.h" +#include "services/network/public/mojom/url_response_head.mojom.h" + +namespace lite_video { + +LiteVideoHintAgent::LiteVideoHintAgent(content::RenderFrame* render_frame) + : content::RenderFrameObserver(render_frame), + content::RenderFrameObserverTracker<LiteVideoHintAgent>(render_frame) { + DCHECK(render_frame); +} + +LiteVideoHintAgent::~LiteVideoHintAgent() = default; + +void LiteVideoHintAgent::OnDestruct() { + delete this; +} + +void LiteVideoHintAgent::AddThrottle(LiteVideoURLLoaderThrottle* throttle) { + DCHECK(HasLiteVideoHint()); + active_throttles_.insert(throttle); + UMA_HISTOGRAM_COUNTS("LiteVideo.HintAgent.ActiveThrottleSize", + active_throttles_.size()); +} + +void LiteVideoHintAgent::RemoveThrottle(LiteVideoURLLoaderThrottle* throttle) { + active_throttles_.erase(throttle); +} + +base::TimeDelta LiteVideoHintAgent::CalculateLatencyForResourceResponse( + const network::mojom::URLResponseHead& response_head) { + if (!HasLiteVideoHint()) + return base::TimeDelta(); + + int64_t recv_bytes = response_head.content_length; + if (recv_bytes == -1) + recv_bytes = response_head.encoded_body_length; + if (recv_bytes == -1) + return base::TimeDelta(); + + if (kilobytes_buffered_before_throttle_ < + *kilobytes_to_buffer_before_throttle_) { + kilobytes_buffered_before_throttle_ += recv_bytes / 1024; + return base::TimeDelta(); + } + + // The total RTT for this media response should be based on how much time it + // took to transfer the packet in the target bandwidth, and the per RTT + // latency. For example, assuming 100KBPS target bandwidth and target RTT of 1 + // second, an 400KB response should have total delay of 5 seconds + // (400/100 + 1). + auto delay_for_throttled_response = + base::TimeDelta::FromSecondsD( + recv_bytes / (*target_downlink_bandwidth_kbps_ * 1024.0)) + + *target_downlink_rtt_latency_; + auto response_delay = + response_head.response_time - response_head.request_time; + if (delay_for_throttled_response <= response_delay) + return base::TimeDelta(); + + return std::min(delay_for_throttled_response - response_delay, + *max_throttling_delay_); +} + +bool LiteVideoHintAgent::HasLiteVideoHint() const { + return target_downlink_bandwidth_kbps_ && target_downlink_rtt_latency_ && + kilobytes_to_buffer_before_throttle_ && max_throttling_delay_; +} + +void LiteVideoHintAgent::SetLiteVideoHint( + blink::mojom::LiteVideoHintPtr lite_video_hint) { + if (!lite_video_hint) + return; + target_downlink_bandwidth_kbps_ = + lite_video_hint->target_downlink_bandwidth_kbps; + kilobytes_to_buffer_before_throttle_ = + lite_video_hint->kilobytes_to_buffer_before_throttle; + target_downlink_rtt_latency_ = lite_video_hint->target_downlink_rtt_latency; + max_throttling_delay_ = lite_video_hint->max_throttling_delay; + LOCAL_HISTOGRAM_BOOLEAN("LiteVideo.HintAgent.HasHint", true); +} + +void LiteVideoHintAgent::StopThrottling() { + // TODO(rajendrant): Send the stop throttling signal to browser process, after + // some K rebuffer events had occurred. + DCHECK(HasLiteVideoHint()); + for (auto* throttle : active_throttles_) { + throttle->ResumeIfThrottled(); + } + kilobytes_buffered_before_throttle_ = 0; +} + +} // namespace lite_video diff --git a/chromium/chrome/renderer/lite_video/lite_video_hint_agent.h b/chromium/chrome/renderer/lite_video/lite_video_hint_agent.h new file mode 100644 index 00000000000..023150bb979 --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_hint_agent.h @@ -0,0 +1,94 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_HINT_AGENT_H_ +#define CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_HINT_AGENT_H_ + +#include "base/macros.h" +#include "base/time/time.h" +#include "chrome/renderer/lite_video/lite_video_url_loader_throttle.h" +#include "content/public/renderer/render_frame_observer.h" +#include "content/public/renderer/render_frame_observer_tracker.h" +#include "third_party/blink/public/mojom/loader/previews_resource_loading_hints.mojom.h" +#include "url/gurl.h" + +namespace lite_video { + +// The renderer-side agent for LiteVideos. There is one instance per frame (main +// frame and subframes), to receive LiteVideo throttling parameters from +// browser. +class LiteVideoHintAgent + : public content::RenderFrameObserver, + public content::RenderFrameObserverTracker<LiteVideoHintAgent> { + public: + explicit LiteVideoHintAgent(content::RenderFrame* render_frame); + ~LiteVideoHintAgent() override; + + LiteVideoHintAgent(const LiteVideoHintAgent&) = delete; + LiteVideoHintAgent& operator=(const LiteVideoHintAgent&) = delete; + + // Returns how much time the media response should get throttled. This is the + // difference between the target latency based on target bandwidth, RTT, and + // the latency the response has already spent. Empty duration is returned when + // the response should not be throttled. The first + // |kilobytes_buffered_before_throttle_| for this render frame should not be + // throttled. This function also updates + // |kilobytes_buffered_before_throttle_|. + base::TimeDelta CalculateLatencyForResourceResponse( + const network::mojom::URLResponseHead& response_head); + + // Updates the LiteVideo throttling parameters for calculating + // the latency to add to media requests. + void SetLiteVideoHint(blink::mojom::LiteVideoHintPtr lite_video_hint); + + // Returns whether |this| has been provided a LiteVideoHint and + // has the parameters needed for calculating the throttling latency. + bool HasLiteVideoHint() const; + + void AddThrottle(LiteVideoURLLoaderThrottle* throttle); + void RemoveThrottle(LiteVideoURLLoaderThrottle* throttle); + + const std::set<LiteVideoURLLoaderThrottle*>& GetActiveThrottlesForTesting() + const { + return active_throttles_; + } + + // Stop throttling and resume the current throttled media requests + // immediately. Throttling could start again for new requests + void StopThrottling(); + + private: + friend class LiteVideoHintAgentTest; + + // content::RenderFrameObserver overrides + void OnDestruct() override; + + // The network downlink bandwidth target in kilobytes per second used to + // calculate the throttling delay on media requests + base::Optional<int> target_downlink_bandwidth_kbps_; + + // The network downlink rtt target latency used to calculate the + // throttling delay on media requests + base::Optional<base::TimeDelta> target_downlink_rtt_latency_; + + // The number of kilobytes for media to be observed before starting to + // throttle requests. + base::Optional<int> kilobytes_to_buffer_before_throttle_; + + // The maximum delay a throttle can introduce for a media request in + // milliseconds. + base::Optional<base::TimeDelta> max_throttling_delay_; + + // The number of media KB that have been left unthrottled before starting + // to introduce a throttling delay. + int kilobytes_buffered_before_throttle_ = 0; + + // Set of media requests that are throttled currently. These are maintained + // here to resume them immediately upon StopThrottling() + std::set<LiteVideoURLLoaderThrottle*> active_throttles_; +}; + +} // namespace lite_video + +#endif // CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_HINT_AGENT_H_ diff --git a/chromium/chrome/renderer/lite_video/lite_video_hint_agent_browsertest.cc b/chromium/chrome/renderer/lite_video/lite_video_hint_agent_browsertest.cc new file mode 100644 index 00000000000..b22f60d9b5d --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_hint_agent_browsertest.cc @@ -0,0 +1,236 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/renderer/lite_video/lite_video_hint_agent.h" + +#include <memory> + +#include "base/macros.h" +#include "base/test/metrics/histogram_tester.h" +#include "base/test/scoped_feature_list.h" +#include "chrome/common/chrome_features.h" +#include "chrome/renderer/lite_video/lite_video_url_loader_throttle.h" +#include "chrome/test/base/chrome_render_view_test.h" +#include "chrome/test/base/ui_test_utils.h" +#include "content/public/renderer/render_frame.h" +#include "content/public/renderer/render_view.h" +#include "services/network/test/test_utils.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/blink/public/platform/web_network_state_notifier.h" + +namespace lite_video { + +namespace { +constexpr char kTestURL[] = "https://litevideo.test.com"; + +} + +// Encapsulates the media URLLoader throttle, its delegate, and maintains the +// current throttling state. +class MediaLoaderThrottleInfo : public blink::URLLoaderThrottle::Delegate { + public: + explicit MediaLoaderThrottleInfo( + std::unique_ptr<LiteVideoURLLoaderThrottle> throttle) + : throttle_(std::move(throttle)) { + throttle_->set_delegate(this); + } + + void SendResponse(network::mojom::URLResponseHead* response_head) { + throttle_->WillProcessResponse(GURL(kTestURL), response_head, + &is_throttled_); + } + + // Implements blink::URLLoaderThrottle::Delegate. + void CancelWithError(int error_code, + base::StringPiece custom_reason) override { + NOTIMPLEMENTED(); + } + void Resume() override { + ASSERT_TRUE(is_throttled_); + is_throttled_ = false; + } + + bool is_throttled() { return is_throttled_; } + + private: + // Current throttling state. + bool is_throttled_ = false; + + std::unique_ptr<LiteVideoURLLoaderThrottle> throttle_; +}; + +class LiteVideoHintAgentTest : public ChromeRenderViewTest { + public: + void DisableLiteVideoFeature() { + scoped_feature_list_.Reset(); + scoped_feature_list_.InitAndDisableFeature(features::kLiteVideo); + } + + std::unique_ptr<LiteVideoURLLoaderThrottle> CreateLiteVideoURLLoaderThrottle( + blink::mojom::RequestContextType request_context_type) { + blink::WebURLRequest request; + request.SetUrl(GURL(kTestURL)); + request.SetRequestContext(request_context_type); + return LiteVideoURLLoaderThrottle::MaybeCreateThrottle( + request, view_->GetMainRenderFrame()->GetRoutingID()); + } + + std::unique_ptr<MediaLoaderThrottleInfo> CreateThrottleAndSendResponse( + net::HttpStatusCode response_code, + const std::string& mime_type, + int content_length) { + auto throttle_info = std::make_unique<MediaLoaderThrottleInfo>( + CreateLiteVideoURLLoaderThrottle( + blink::mojom::RequestContextType::FETCH)); + network::mojom::URLResponseHeadPtr response_head = + network::CreateURLResponseHead(response_code); + response_head->mime_type = mime_type; + response_head->mime_type = mime_type; + response_head->content_length = content_length; + response_head->network_accessed = true; + response_head->was_fetched_via_cache = false; + + throttle_info->SendResponse(response_head.get()); + return throttle_info; + } + + const std::set<LiteVideoURLLoaderThrottle*>& GetActiveThrottledResponses() + const { + return lite_video_hint_agent_->GetActiveThrottlesForTesting(); + } + + const base::HistogramTester& histogram_tester() { return histogram_tester_; } + + void StopThrottling() { lite_video_hint_agent_->StopThrottling(); } + + protected: + void SetUp() override { + ChromeRenderViewTest::SetUp(); + scoped_feature_list_.InitAndEnableFeature(features::kLiteVideo); + blink::WebNetworkStateNotifier::SetSaveDataEnabled(true); + lite_video_hint_agent_ = + new LiteVideoHintAgent(view_->GetMainRenderFrame()); + + // Set some default hints. + blink::mojom::LiteVideoHintPtr hint = blink::mojom::LiteVideoHint::New(); + hint->kilobytes_to_buffer_before_throttle = 10; + hint->target_downlink_bandwidth_kbps = 60; + hint->target_downlink_rtt_latency = base::TimeDelta::FromMilliseconds(500); + hint->max_throttling_delay = base::TimeDelta::FromSeconds(5); + lite_video_hint_agent_->SetLiteVideoHint(std::move(hint)); + } + + private: + base::test::ScopedFeatureList scoped_feature_list_; + + base::HistogramTester histogram_tester_; + + // Owned by the RenderFrame. + LiteVideoHintAgent* lite_video_hint_agent_; +}; + +TEST_F(LiteVideoHintAgentTest, LiteVideoDisabled) { + DisableLiteVideoFeature(); + EXPECT_FALSE(CreateLiteVideoURLLoaderThrottle( + blink::mojom::RequestContextType::FETCH)); +} + +TEST_F(LiteVideoHintAgentTest, SaveDataDisabled) { + blink::WebNetworkStateNotifier::SetSaveDataEnabled(false); + EXPECT_FALSE(CreateLiteVideoURLLoaderThrottle( + blink::mojom::RequestContextType::FETCH)); +} + +TEST_F(LiteVideoHintAgentTest, OnlyFetchAPIResponseThrottled) { + EXPECT_FALSE(CreateLiteVideoURLLoaderThrottle( + blink::mojom::RequestContextType::IMAGE)); +} + +TEST_F(LiteVideoHintAgentTest, OnlyMediaMimeTypeThrottled) { + histogram_tester().ExpectUniqueSample("LiteVideo.HintAgent.HasHint", true, 1); + + auto throttle_info = + CreateThrottleAndSendResponse(net::HTTP_OK, "image/jpeg", 11000); + EXPECT_FALSE(throttle_info->is_throttled()); + + throttle_info = + CreateThrottleAndSendResponse(net::HTTP_OK, "image/jpeg", 11000); + EXPECT_FALSE(throttle_info->is_throttled()); + histogram_tester().ExpectTotalCount("LiteVideo.URLLoader.ThrottleLatency", 0); +} + +TEST_F(LiteVideoHintAgentTest, FailedMediaResponseNotThrottled) { + histogram_tester().ExpectUniqueSample("LiteVideo.HintAgent.HasHint", true, 1); + + auto throttle_info = CreateThrottleAndSendResponse( + net::HTTP_INTERNAL_SERVER_ERROR, "video/mp4", 11000); + EXPECT_FALSE(throttle_info->is_throttled()); + + throttle_info = CreateThrottleAndSendResponse(net::HTTP_INTERNAL_SERVER_ERROR, + "video/mp4", 11000); + EXPECT_FALSE(throttle_info->is_throttled()); + histogram_tester().ExpectTotalCount("LiteVideo.URLLoader.ThrottleLatency", 0); +} + +TEST_F(LiteVideoHintAgentTest, MediaResponseThrottled) { + histogram_tester().ExpectUniqueSample("LiteVideo.HintAgent.HasHint", true, 1); + + // Initial k media bytes will not be throttled. + auto throttle_info = + CreateThrottleAndSendResponse(net::HTTP_OK, "video/mp4", 11000); + EXPECT_FALSE(throttle_info->is_throttled()); + histogram_tester().ExpectTotalCount("LiteVideo.URLLoader.ThrottleLatency", 0); + EXPECT_TRUE(GetActiveThrottledResponses().empty()); + + // Verify if a response gets throttled and eventually resumed. + throttle_info = + CreateThrottleAndSendResponse(net::HTTP_OK, "video/mp4", 440000); + histogram_tester().ExpectTotalCount("LiteVideo.URLLoader.ThrottleLatency", 1); + EXPECT_TRUE(throttle_info->is_throttled()); + // This is to wait until the throttle resumes, cannot fast-forward in + // RenderViewTest. + while (throttle_info->is_throttled()) + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(throttle_info->is_throttled()); + + // Verify a response that wasn't yet resumed, gets cleared from hint agent + // when its destroyed. + throttle_info = + CreateThrottleAndSendResponse(net::HTTP_OK, "video/mp4", 440000); + EXPECT_TRUE(throttle_info->is_throttled()); + histogram_tester().ExpectTotalCount("LiteVideo.URLLoader.ThrottleLatency", 2); + + EXPECT_FALSE(GetActiveThrottledResponses().empty()); + throttle_info.reset(); + EXPECT_TRUE(GetActiveThrottledResponses().empty()); +} + +TEST_F(LiteVideoHintAgentTest, StopThrottlingResumesResponsesImmediately) { + histogram_tester().ExpectUniqueSample("LiteVideo.HintAgent.HasHint", true, 1); + + // Initial response is not throttled, and the next two are throttled. + auto throttle_info1 = + CreateThrottleAndSendResponse(net::HTTP_OK, "video/mp4", 11000); + auto throttle_info2 = + CreateThrottleAndSendResponse(net::HTTP_OK, "video/mp4", 11000); + auto throttle_info3 = + CreateThrottleAndSendResponse(net::HTTP_OK, "video/mp4", 11000); + EXPECT_FALSE(throttle_info1->is_throttled()); + EXPECT_TRUE(throttle_info2->is_throttled()); + EXPECT_TRUE(throttle_info3->is_throttled()); + histogram_tester().ExpectTotalCount("LiteVideo.URLLoader.ThrottleLatency", 2); + EXPECT_EQ(2U, GetActiveThrottledResponses().size()); + + // Stop throttling will immediately resume. + StopThrottling(); + EXPECT_FALSE(throttle_info2->is_throttled()); + EXPECT_FALSE(throttle_info3->is_throttled()); + + // When the throttle destroys it should get removed from active throttles. + throttle_info2.reset(); + throttle_info3.reset(); + EXPECT_TRUE(GetActiveThrottledResponses().empty()); +} + +} // namespace lite_video diff --git a/chromium/chrome/renderer/lite_video/lite_video_url_loader_throttle.cc b/chromium/chrome/renderer/lite_video/lite_video_url_loader_throttle.cc new file mode 100644 index 00000000000..acde3638f64 --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_url_loader_throttle.cc @@ -0,0 +1,121 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/renderer/lite_video/lite_video_url_loader_throttle.h" + +#include "base/metrics/histogram_macros.h" +#include "base/test/metrics/histogram_tester.h" +#include "chrome/renderer/lite_video/lite_video_hint_agent.h" +#include "chrome/renderer/lite_video/lite_video_util.h" +#include "content/public/renderer/render_frame.h" +#include "content/public/renderer/render_thread.h" +#include "services/network/public/mojom/url_response_head.mojom.h" + +namespace lite_video { + +LiteVideoHintAgent* GetLiteVideoHintAgent(int render_frame_id) { + DCHECK_NE(MSG_ROUTING_NONE, render_frame_id); + if (auto* render_frame = + content::RenderFrame::FromRoutingID(render_frame_id)) { + return LiteVideoHintAgent::Get(render_frame); + } + return nullptr; +} + +// static +std::unique_ptr<LiteVideoURLLoaderThrottle> +LiteVideoURLLoaderThrottle::MaybeCreateThrottle( + const blink::WebURLRequest& request, + int render_frame_id) { + auto request_context = request.GetRequestContext(); + if (request_context != blink::mojom::RequestContextType::FETCH && + request_context != blink::mojom::RequestContextType::XML_HTTP_REQUEST) { + return nullptr; + } + // TODO(rajendrant): Also allow the throttle to be stopped when LiteMode gets + // disabled or ECT worsens. This logic should probably be in the browser + // process. + if (!IsLiteVideoEnabled()) + return nullptr; + + auto* lite_video_hint_agent = GetLiteVideoHintAgent(render_frame_id); + if (lite_video_hint_agent && lite_video_hint_agent->HasLiteVideoHint()) + return std::make_unique<LiteVideoURLLoaderThrottle>(render_frame_id); + + return nullptr; +} + +LiteVideoURLLoaderThrottle::LiteVideoURLLoaderThrottle(int render_frame_id) + : render_frame_id_(render_frame_id) { + DCHECK(IsLiteVideoEnabled()); +} + +LiteVideoURLLoaderThrottle::~LiteVideoURLLoaderThrottle() { + // Existence of |response_delay_timer_| indicates throttling has been + // attempted on this media response. Remove the throttle on this case. + if (response_delay_timer_) { + DCHECK(render_frame_id_); + auto* lite_video_hint_agent = GetLiteVideoHintAgent(render_frame_id_); + if (lite_video_hint_agent) + lite_video_hint_agent->RemoveThrottle(this); + } +} + +void LiteVideoURLLoaderThrottle::WillProcessResponse( + const GURL& response_url, + network::mojom::URLResponseHead* response_head, + bool* defer) { + if (!response_head || !response_head->headers) + return; + // Do not throttle on 4xx, 5xx failures. + if (response_head->headers->response_code() != 200) + return; + if (!response_head->network_accessed || + response_head->was_fetched_via_cache) { + return; + } + if (!base::StartsWith(response_head->mime_type, "video/", + base::CompareCase::SENSITIVE)) { + return; + } + + auto* lite_video_hint_agent = GetLiteVideoHintAgent(render_frame_id_); + if (!lite_video_hint_agent) + return; + + auto latency = lite_video_hint_agent->CalculateLatencyForResourceResponse( + *response_head); + if (latency.is_zero()) + return; + + UMA_HISTOGRAM_TIMES("LiteVideo.URLLoader.ThrottleLatency", latency); + + *defer = true; + // The timer may have already started and running, and the below restart will + // lose that elapsed time. However the elapsed time will be small since this + // case happens if some other url loader throttle is restarting the request. + response_delay_timer_ = std::make_unique<base::OneShotTimer>(); + response_delay_timer_->Start( + FROM_HERE, latency, + base::BindOnce(&LiteVideoURLLoaderThrottle::ResumeThrottledMediaResponse, + base::Unretained(this))); + + lite_video_hint_agent->AddThrottle(this); +} + +void LiteVideoURLLoaderThrottle::ResumeIfThrottled() { + if (response_delay_timer_ && response_delay_timer_->IsRunning()) { + response_delay_timer_->Stop(); + ResumeThrottledMediaResponse(); + } +} + +void LiteVideoURLLoaderThrottle::ResumeThrottledMediaResponse() { + DCHECK(!response_delay_timer_->IsRunning()); + delegate_->Resume(); +} + +void LiteVideoURLLoaderThrottle::DetachFromCurrentSequence() {} + +} // namespace lite_video diff --git a/chromium/chrome/renderer/lite_video/lite_video_url_loader_throttle.h b/chromium/chrome/renderer/lite_video/lite_video_url_loader_throttle.h new file mode 100644 index 00000000000..ec353aa16c4 --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_url_loader_throttle.h @@ -0,0 +1,56 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_URL_LOADER_THROTTLE_H_ +#define CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_URL_LOADER_THROTTLE_H_ + +#include "base/callback.h" +#include "base/timer/timer.h" +#include "third_party/blink/public/common/loader/url_loader_throttle.h" + +namespace blink { +class WebURLRequest; +} // namespace blink + +namespace lite_video { + +// This class throttles media url requests that have audio/video mime-type in +// response headers, to simulate low bandwidth conditions. This allows MSE video +// players to adapt and play the lower resolution videos. +class LiteVideoURLLoaderThrottle : public blink::URLLoaderThrottle { + public: + explicit LiteVideoURLLoaderThrottle(int render_frame_id); + ~LiteVideoURLLoaderThrottle() override; + + // Creates throttle for |request| if LiteVideo is enabled for LiteMode users, + // and LiteVideoHintAgent has received hints for the navigation. Throttle will + // be created only for Fetch/XHR requests. + static std::unique_ptr<LiteVideoURLLoaderThrottle> MaybeCreateThrottle( + const blink::WebURLRequest& request, + int render_frame_id); + + // Resumes the media response if it was currently throttled. Otherwise its a + // no-op. + void ResumeIfThrottled(); + + // blink::URLLoaderThrottle: + void WillProcessResponse(const GURL& response_url, + network::mojom::URLResponseHead* response_head, + bool* defer) override; + void DetachFromCurrentSequence() override; + + private: + // Resumes the media response immediately. + void ResumeThrottledMediaResponse(); + + // Render frame id to get the media throttle observer of the render frame. + const int render_frame_id_; + + // Timer to introduce latency for the response. + std::unique_ptr<base::OneShotTimer> response_delay_timer_; +}; + +} // namespace lite_video + +#endif // CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_URL_LOADER_THROTTLE_H_ diff --git a/chromium/chrome/renderer/lite_video/lite_video_util.cc b/chromium/chrome/renderer/lite_video/lite_video_util.cc new file mode 100644 index 00000000000..69c483cb1a1 --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_util.cc @@ -0,0 +1,17 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "chrome/renderer/lite_video/lite_video_util.h" + +#include "chrome/common/chrome_features.h" +#include "third_party/blink/public/platform/web_network_state_notifier.h" + +namespace lite_video { + +bool IsLiteVideoEnabled() { + return base::FeatureList::IsEnabled(features::kLiteVideo) && + blink::WebNetworkStateNotifier::SaveDataEnabled(); +} + +} // namespace lite_video diff --git a/chromium/chrome/renderer/lite_video/lite_video_util.h b/chromium/chrome/renderer/lite_video/lite_video_util.h new file mode 100644 index 00000000000..35c4603be9d --- /dev/null +++ b/chromium/chrome/renderer/lite_video/lite_video_util.h @@ -0,0 +1,15 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_UTIL_H_ +#define CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_UTIL_H_ + +namespace lite_video { + +// Returns whether LiteVideo is enabled. +bool IsLiteVideoEnabled(); + +} // namespace lite_video + +#endif // CHROME_RENDERER_LITE_VIDEO_LITE_VIDEO_UTIL_H_ diff --git a/chromium/chrome/renderer/media/OWNERS b/chromium/chrome/renderer/media/OWNERS index e5a3f968d1c..3de875bc8e3 100644 --- a/chromium/chrome/renderer/media/OWNERS +++ b/chromium/chrome/renderer/media/OWNERS @@ -2,10 +2,6 @@ file://media/OWNERS sergeyu@chromium.org tommi@chromium.org -# For Cast-related changes. -per-file cast_*=miu@chromium.org -per-file cast_*=mfoltz@chromium.org - # FlashEmbedRewrite per-file flash_embed_rewrite*=mlamouri@chromium.org diff --git a/chromium/chrome/renderer/media/cast_ipc_dispatcher.cc b/chromium/chrome/renderer/media/cast_ipc_dispatcher.cc deleted file mode 100644 index 3025cc4fbc7..00000000000 --- a/chromium/chrome/renderer/media/cast_ipc_dispatcher.cc +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_ipc_dispatcher.h" - -#include "base/single_thread_task_runner.h" -#include "chrome/common/cast_messages.h" -#include "chrome/renderer/media/cast_transport_ipc.h" -#include "ipc/ipc_message_macros.h" - -CastIPCDispatcher* CastIPCDispatcher::global_instance_ = NULL; - -CastIPCDispatcher::CastIPCDispatcher( - const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner) - : sender_(NULL), - io_task_runner_(io_task_runner) { - DCHECK(io_task_runner_.get()); - DCHECK(!global_instance_); -} - -CastIPCDispatcher::~CastIPCDispatcher() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK(!global_instance_); -} - -CastIPCDispatcher* CastIPCDispatcher::Get() { - return global_instance_; -} - -void CastIPCDispatcher::Send(IPC::Message* message) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - if (sender_) { - sender_->Send(message); - } else { - delete message; - } -} - -int32_t CastIPCDispatcher::AddSender(CastTransportIPC* sender) { - return id_map_.Add(sender); -} - -void CastIPCDispatcher::RemoveSender(int32_t channel_id) { - return id_map_.Remove(channel_id); -} - -bool CastIPCDispatcher::OnMessageReceived(const IPC::Message& message) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - bool handled = true; - IPC_BEGIN_MESSAGE_MAP(CastIPCDispatcher, message) - IPC_MESSAGE_HANDLER(CastMsg_NotifyStatusChange, OnNotifyStatusChange) - IPC_MESSAGE_HANDLER(CastMsg_RawEvents, OnRawEvents) - IPC_MESSAGE_HANDLER(CastMsg_Rtt, OnRtt) - IPC_MESSAGE_HANDLER(CastMsg_RtcpCastMessage, OnRtcpCastMessage) - IPC_MESSAGE_HANDLER(CastMsg_Pli, OnReceivedPli); - IPC_MESSAGE_HANDLER(CastMsg_ReceivedPacket, OnReceivedPacket) - IPC_MESSAGE_UNHANDLED(handled = false) - IPC_END_MESSAGE_MAP() - return handled; -} - -void CastIPCDispatcher::OnFilterAdded(IPC::Channel* channel) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK(!global_instance_); - global_instance_ = this; - sender_ = channel; -} - -void CastIPCDispatcher::OnFilterRemoved() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK_EQ(this, global_instance_); - global_instance_ = NULL; - sender_ = NULL; -} - -void CastIPCDispatcher::OnChannelClosing() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - DCHECK_EQ(this, global_instance_); -} - -void CastIPCDispatcher::OnNotifyStatusChange( - int32_t channel_id, - media::cast::CastTransportStatus status) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnNotifyStatusChange(status); - } else { - DVLOG(1) - << "CastIPCDispatcher::OnNotifystatusChange on non-existing channel."; - } -} - -void CastIPCDispatcher::OnRawEvents( - int32_t channel_id, - const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnRawEvents(packet_events, frame_events); - } else { - DVLOG(1) << "CastIPCDispatcher::OnRawEvents on non-existing channel."; - } -} - -void CastIPCDispatcher::OnRtt(int32_t channel_id, - uint32_t ssrc, - base::TimeDelta rtt) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnRtt(ssrc, rtt); - } else { - DVLOG(1) << "CastIPCDispatcher::OnRtt on non-existing channel."; - } -} - -void CastIPCDispatcher::OnRtcpCastMessage( - int32_t channel_id, - uint32_t ssrc, - const media::cast::RtcpCastMessage& cast_message) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnRtcpCastMessage(ssrc, cast_message); - } else { - DVLOG(1) << "CastIPCDispatcher::OnRtt on non-existing channel."; - } -} - -void CastIPCDispatcher::OnReceivedPli(int32_t channel_id, int32_t ssrc) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnReceivedPli(ssrc); - } else { - DVLOG(1) << "CastIPCDispatcher::OnReceivedPli on non-existing " - "channel."; - } -} - -void CastIPCDispatcher::OnReceivedPacket(int32_t channel_id, - const media::cast::Packet& packet) { - CastTransportIPC* sender = id_map_.Lookup(channel_id); - if (sender) { - sender->OnReceivedPacket(packet); - } else { - DVLOG(1) << "CastIPCDispatcher::OnReceievdPacket on non-existing channel."; - } -} diff --git a/chromium/chrome/renderer/media/cast_ipc_dispatcher.h b/chromium/chrome/renderer/media/cast_ipc_dispatcher.h deleted file mode 100644 index 32f898fddb2..00000000000 --- a/chromium/chrome/renderer/media/cast_ipc_dispatcher.h +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_IPC_DISPATCHER_H_ -#define CHROME_RENDERER_MEDIA_CAST_IPC_DISPATCHER_H_ - -#include <stdint.h> - -#include "base/callback.h" -#include "base/containers/id_map.h" -#include "base/macros.h" -#include "base/threading/thread_task_runner_handle.h" -#include "ipc/ipc_channel_proxy.h" -#include "ipc/message_filter.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/net/cast_transport.h" - -class CastTransportIPC; - -// This dispatcher listens to incoming IPC messages and sends -// the call to the correct CastTransportIPC instance. -class CastIPCDispatcher : public IPC::MessageFilter { - public: - explicit CastIPCDispatcher( - const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner); - - static CastIPCDispatcher* Get(); - void Send(IPC::Message* message); - int32_t AddSender(CastTransportIPC* sender); - void RemoveSender(int32_t channel_id); - - // IPC::MessageFilter implementation - bool OnMessageReceived(const IPC::Message& message) override; - void OnFilterAdded(IPC::Channel* channel) override; - void OnFilterRemoved() override; - void OnChannelClosing() override; - - protected: - ~CastIPCDispatcher() override; - - private: - void OnNotifyStatusChange(int32_t channel_id, - media::cast::CastTransportStatus status); - void OnRtpStatistics(int32_t channel_id, - bool audio, - const media::cast::RtcpSenderInfo& sender_info, - base::TimeTicks time_sent, - uint32_t rtp_timestamp); - void OnRawEvents(int32_t channel_id, - const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events); - void OnRtt(int32_t channel_id, uint32_t ssrc, base::TimeDelta rtt); - void OnRtcpCastMessage(int32_t channel_id, - uint32_t ssrc, - const media::cast::RtcpCastMessage& cast_message); - void OnReceivedPli(int32_t channel_id, int32_t ssrc); - void OnReceivedPacket(int32_t channel_id, const media::cast::Packet& packet); - - static CastIPCDispatcher* global_instance_; - - // For IPC Send(); must only be accesed on |io_message_loop_|. - IPC::Sender* sender_; - - // Task runner on which IPC calls are driven. - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - - // A map of stream ids to delegates; must only be accessed on - // |io_message_loop_|. - base::IDMap<CastTransportIPC*> id_map_; - DISALLOW_COPY_AND_ASSIGN(CastIPCDispatcher); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_IPC_DISPATCHER_H_ diff --git a/chromium/chrome/renderer/media/cast_ipc_dispatcher_unittest.cc b/chromium/chrome/renderer/media/cast_ipc_dispatcher_unittest.cc deleted file mode 100644 index 7dd002ae427..00000000000 --- a/chromium/chrome/renderer/media/cast_ipc_dispatcher_unittest.cc +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_ipc_dispatcher.h" - -#include "base/test/simple_test_tick_clock.h" -#include "base/test/task_environment.h" -#include "base/threading/thread_task_runner_handle.h" -#include "chrome/common/cast_messages.h" -#include "ipc/ipc_message_macros.h" -#include "media/cast/logging/logging_defines.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace { - -class CastIPCDispatcherTest : public testing::Test { - public: - CastIPCDispatcherTest() { - dispatcher_ = new CastIPCDispatcher(base::ThreadTaskRunnerHandle::Get()); - } - - protected: - void FakeSend(const IPC::Message& message) { - EXPECT_TRUE(dispatcher_->OnMessageReceived(message)); - } - - scoped_refptr<CastIPCDispatcher> dispatcher_; - base::test::SingleThreadTaskEnvironment task_environment_{ - base::test::SingleThreadTaskEnvironment::MainThreadType::IO}; -}; - -TEST_F(CastIPCDispatcherTest, RawEvents) { - const int kChannelId = 17; - - media::cast::PacketEvent packet_event; - packet_event.rtp_timestamp = - media::cast::RtpTimeTicks().Expand(UINT32_C(100)); - packet_event.max_packet_id = 10; - packet_event.packet_id = 5; - packet_event.size = 512; - packet_event.timestamp = base::SimpleTestTickClock().NowTicks(); - packet_event.type = media::cast::PACKET_SENT_TO_NETWORK; - packet_event.media_type = media::cast::VIDEO_EVENT; - std::vector<media::cast::PacketEvent> packet_events; - packet_events.push_back(packet_event); - - media::cast::FrameEvent frame_event; - frame_event.rtp_timestamp = media::cast::RtpTimeTicks().Expand(UINT32_C(100)); - frame_event.frame_id = media::cast::FrameId::first() + 5; - frame_event.size = 512; - frame_event.timestamp = base::SimpleTestTickClock().NowTicks(); - frame_event.media_type = media::cast::VIDEO_EVENT; - std::vector<media::cast::FrameEvent> frame_events; - frame_events.push_back(frame_event); - - packet_events.push_back(packet_event); - CastMsg_RawEvents raw_events_msg(kChannelId, packet_events, - frame_events); - - FakeSend(raw_events_msg); -} - -} // namespace diff --git a/chromium/chrome/renderer/media/cast_receiver_audio_valve.cc b/chromium/chrome/renderer/media/cast_receiver_audio_valve.cc deleted file mode 100644 index c2d5b856727..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_audio_valve.cc +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_receiver_audio_valve.h" - -#include "base/bind.h" -#include "base/bind_helpers.h" -#include "media/base/audio_parameters.h" - -CastReceiverAudioValve::CastReceiverAudioValve( - const media::AudioParameters& params, - media::AudioCapturerSource::CaptureCallback* cb) - : cb_(cb), - fifo_(base::Bind(&CastReceiverAudioValve::DeliverRebufferedAudio, - base::Unretained(this))), - sample_rate_(params.sample_rate()) { - fifo_.Reset(params.frames_per_buffer()); -} - -CastReceiverAudioValve::~CastReceiverAudioValve() {} - -void CastReceiverAudioValve::DeliverDecodedAudio( - const media::AudioBus* audio_bus, - base::TimeTicks playout_time) { - current_playout_time_ = playout_time; - // The following will result in zero, one, or multiple synchronous calls to - // DeliverRebufferedAudio(). - fifo_.Push(*audio_bus); -} - -void CastReceiverAudioValve::DeliverRebufferedAudio( - const media::AudioBus& audio_bus, - int frame_delay) { - const base::TimeTicks playout_time = - current_playout_time_ + - base::TimeDelta::FromMicroseconds( - frame_delay * base::Time::kMicrosecondsPerSecond / sample_rate_); - - base::AutoLock lock(lock_); - if (cb_) { - cb_->Capture(&audio_bus, playout_time, 1.0 /* volume */, - false /* key_pressed */); - } -} - -void CastReceiverAudioValve::Stop() { - base::AutoLock lock(lock_); - cb_ = nullptr; -} diff --git a/chromium/chrome/renderer/media/cast_receiver_audio_valve.h b/chromium/chrome/renderer/media/cast_receiver_audio_valve.h deleted file mode 100644 index f6c09a74a26..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_audio_valve.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RECEIVER_AUDIO_VALVE_H_ -#define CHROME_RENDERER_MEDIA_CAST_RECEIVER_AUDIO_VALVE_H_ - -#include "base/synchronization/lock.h" -#include "base/time/time.h" -#include "media/base/audio_capturer_source.h" -#include "media/base/audio_push_fifo.h" - -namespace media { -class AudioBus; -} - -// Forwards calls to |cb| until Stop is called. If the client requested a -// different buffer size than that provided by the Cast Receiver, AudioPushFifo -// is used to rectify that. -// -// Thread-safe. -// All functions may block depending on contention. -class CastReceiverAudioValve : - public base::RefCountedThreadSafe<CastReceiverAudioValve> { - public: - CastReceiverAudioValve(const media::AudioParameters& params, - media::AudioCapturerSource::CaptureCallback* cb); - - // Called on an unknown thread to provide more decoded audio data from the - // Cast Receiver. - void DeliverDecodedAudio(const media::AudioBus* audio_bus, - base::TimeTicks playout_time); - - // When this returns, no more calls will be forwarded to |cb|. - void Stop(); - - private: - friend class base::RefCountedThreadSafe<CastReceiverAudioValve>; - - ~CastReceiverAudioValve(); - - // Called by AudioPushFifo zero or more times during the call to Capture(). - // Delivers audio data in the required buffer size to |cb_|. - void DeliverRebufferedAudio(const media::AudioBus& audio_bus, - int frame_delay); - - media::AudioCapturerSource::CaptureCallback* cb_; - base::Lock lock_; - - media::AudioPushFifo fifo_; - const int sample_rate_; - - // Used to pass the current playout time between DeliverDecodedAudio() and - // DeviliverRebufferedAudio(). - base::TimeTicks current_playout_time_; -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RECEIVER_AUDIO_VALVE_H_ diff --git a/chromium/chrome/renderer/media/cast_receiver_session.cc b/chromium/chrome/renderer/media/cast_receiver_session.cc deleted file mode 100644 index 659bf04ca4d..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session.cc +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_receiver_session.h" - -#include <memory> - -#include "base/bind.h" -#include "base/location.h" -#include "base/synchronization/waitable_event.h" -#include "base/threading/thread_task_runner_handle.h" -#include "chrome/renderer/media/cast_receiver_audio_valve.h" -#include "content/public/renderer/render_thread.h" -#include "media/base/audio_capturer_source.h" -#include "media/base/bind_to_current_loop.h" -#include "media/capture/video_capturer_source.h" -#include "third_party/blink/public/platform/web_media_stream.h" -#include "third_party/blink/public/platform/web_media_stream_source.h" -#include "third_party/blink/public/platform/web_media_stream_track.h" - -// This is a render thread object. -class CastReceiverSession::AudioCapturerSource : - public media::AudioCapturerSource { - public: - AudioCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session); - void Initialize(const media::AudioParameters& params, - CaptureCallback* callback) override; - void Start() override; - void Stop() override; - void SetVolume(double volume) override; - void SetAutomaticGainControl(bool enable) override; - void SetOutputDeviceForAec(const std::string& output_device_id) override; - - private: - ~AudioCapturerSource() override; - const scoped_refptr<CastReceiverSession> cast_receiver_session_; - scoped_refptr<CastReceiverAudioValve> audio_valve_; -}; - -// This is a render thread object. -class CastReceiverSession::VideoCapturerSource - : public media::VideoCapturerSource { - public: - explicit VideoCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session); - protected: - media::VideoCaptureFormats GetPreferredFormats() override; - void StartCapture(const media::VideoCaptureParams& params, - const VideoCaptureDeliverFrameCB& frame_callback, - const RunningCallback& running_callback) override; - void StopCapture() override; - private: - const scoped_refptr<CastReceiverSession> cast_receiver_session_; -}; - -CastReceiverSession::CastReceiverSession() - : delegate_(new CastReceiverSessionDelegate()), - io_task_runner_(content::RenderThread::Get()->GetIOTaskRunner()) {} - -CastReceiverSession::~CastReceiverSession() { - // We should always be able to delete the object on the IO thread. - CHECK(io_task_runner_->DeleteSoon(FROM_HERE, delegate_.release())); -} - -void CastReceiverSession::Start( - const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& capture_format, - const StartCB& start_callback, - const CastReceiverSessionDelegate::ErrorCallback& error_callback) { - audio_config_ = audio_config; - video_config_ = video_config; - format_ = capture_format; - io_task_runner_->PostTask( - FROM_HERE, base::BindOnce(&CastReceiverSessionDelegate::Start, - base::Unretained(delegate_.get()), audio_config, - video_config, local_endpoint, remote_endpoint, - std::move(options), format_, - media::BindToCurrentLoop(error_callback))); - scoped_refptr<media::AudioCapturerSource> audio( - new CastReceiverSession::AudioCapturerSource(this)); - std::unique_ptr<media::VideoCapturerSource> video( - new CastReceiverSession::VideoCapturerSource(this)); - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(start_callback, audio, std::move(video))); -} - -void CastReceiverSession::StartAudio( - scoped_refptr<CastReceiverAudioValve> audio_valve) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastReceiverSessionDelegate::StartAudio, - base::Unretained(delegate_.get()), audio_valve)); -} - -void CastReceiverSession::StartVideo( - blink::VideoCaptureDeliverFrameCB frame_callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastReceiverSessionDelegate::StartVideo, - base::Unretained(delegate_.get()), frame_callback)); -} - -void CastReceiverSession::StopVideo() { - io_task_runner_->PostTask( - FROM_HERE, base::BindOnce(&CastReceiverSessionDelegate::StopVideo, - base::Unretained(delegate_.get()))); -} - -CastReceiverSession::VideoCapturerSource::VideoCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session) - : cast_receiver_session_(cast_receiver_session) { -} - -media::VideoCaptureFormats -CastReceiverSession::VideoCapturerSource::GetPreferredFormats() { - media::VideoCaptureFormats formats; - if (cast_receiver_session_->format_.IsValid()) - formats.push_back(cast_receiver_session_->format_); - return formats; -} - -void CastReceiverSession::VideoCapturerSource::StartCapture( - const media::VideoCaptureParams& params, - const VideoCaptureDeliverFrameCB& frame_callback, - const RunningCallback& running_callback) { - cast_receiver_session_->StartVideo(frame_callback); - running_callback.Run(true); -} - -void CastReceiverSession::VideoCapturerSource::StopCapture() { - cast_receiver_session_->StopVideo(); -} - -CastReceiverSession::AudioCapturerSource::AudioCapturerSource( - const scoped_refptr<CastReceiverSession> cast_receiver_session) - : cast_receiver_session_(cast_receiver_session) { -} - -CastReceiverSession::AudioCapturerSource::~AudioCapturerSource() { - DCHECK(!audio_valve_); -} - -void CastReceiverSession::AudioCapturerSource::Initialize( - const media::AudioParameters& params, - CaptureCallback* callback) { - // TODO(hubbe): Consider converting the audio to whatever the caller wants. - if (params.sample_rate() != - cast_receiver_session_->audio_config_.rtp_timebase || - params.channels() != cast_receiver_session_->audio_config_.channels) { - callback->OnCaptureError(std::string()); - return; - } - audio_valve_ = new CastReceiverAudioValve(params, callback); -} - -void CastReceiverSession::AudioCapturerSource::Start() { - DCHECK(audio_valve_); - cast_receiver_session_->StartAudio(audio_valve_); -} - -void CastReceiverSession::AudioCapturerSource::Stop() { - audio_valve_->Stop(); - audio_valve_ = nullptr; -} - -void CastReceiverSession::AudioCapturerSource::SetVolume(double volume) { - // not supported -} - -void CastReceiverSession::AudioCapturerSource::SetAutomaticGainControl( - bool enable) { - // not supported -} - -void CastReceiverSession::AudioCapturerSource::SetOutputDeviceForAec( - const std::string& output_device_id) { - // not supported -} diff --git a/chromium/chrome/renderer/media/cast_receiver_session.h b/chromium/chrome/renderer/media/cast_receiver_session.h deleted file mode 100644 index c9bfa5a1181..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_H_ -#define CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_H_ - -#include <memory> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/single_thread_task_runner.h" -#include "chrome/renderer/media/cast_receiver_session_delegate.h" - -namespace media { -class AudioCapturerSource; -struct VideoCaptureFormat; -class VideoCapturerSource; -} - -namespace net { -class IPEndPoint; -} - -namespace base { -class DictionaryValue; -} - -// This a render thread object, all methods, construction and -// destruction must happen on the render thread. -class CastReceiverSession : public base::RefCounted<CastReceiverSession> { - public: - CastReceiverSession(); - - typedef base::Callback<void(scoped_refptr<media::AudioCapturerSource>, - std::unique_ptr<media::VideoCapturerSource>)> - StartCB; - - // Note that the cast receiver will start responding to - // incoming network streams immediately, buffering input until - // StartAudio/StartVideo is called. - // Five first parameters are passed to cast receiver. - // |start_callback| is called when initialization is done. - // TODO(hubbe): Currently the audio component of the returned media - // stream only the exact format that the sender is sending us. - void Start(const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& capture_format, - const StartCB& start_callback, - const CastReceiverSessionDelegate::ErrorCallback& error_callback); - - private: - class VideoCapturerSource; - class AudioCapturerSource; - friend class base::RefCounted<CastReceiverSession>; - virtual ~CastReceiverSession(); - void StartAudio(scoped_refptr<CastReceiverAudioValve> audio_valve); - - void StartVideo(blink::VideoCaptureDeliverFrameCB frame_callback); - // Stop Video callbacks. - // Note that this returns immediately, but callbacks do not stop immediately. - void StopVideo(); - - media::cast::FrameReceiverConfig audio_config_; - media::cast::FrameReceiverConfig video_config_; - std::unique_ptr<CastReceiverSessionDelegate> delegate_; - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - media::VideoCaptureFormat format_; - - DISALLOW_COPY_AND_ASSIGN(CastReceiverSession); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_H_ diff --git a/chromium/chrome/renderer/media/cast_receiver_session_delegate.cc b/chromium/chrome/renderer/media/cast_receiver_session_delegate.cc deleted file mode 100644 index c6b6a1053b9..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session_delegate.cc +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_receiver_session_delegate.h" - -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/synchronization/waitable_event.h" -#include "base/values.h" - -CastReceiverSessionDelegate::CastReceiverSessionDelegate() {} -CastReceiverSessionDelegate::~CastReceiverSessionDelegate() {} - -void CastReceiverSessionDelegate::Start( - const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& format, - const ErrorCallback& error_callback) { - format_ = format; - DCHECK(io_task_runner_->BelongsToCurrentThread()); - CastSessionDelegateBase::StartUDP(local_endpoint, remote_endpoint, - std::move(options), error_callback); - cast_receiver_ = media::cast::CastReceiver::Create(cast_environment_, - audio_config, - video_config, - cast_transport_.get()); - on_audio_decoded_cb_ = - base::BindRepeating(&CastReceiverSessionDelegate::OnDecodedAudioFrame, - weak_factory_.GetWeakPtr()); - on_video_decoded_cb_ = - base::BindRepeating(&CastReceiverSessionDelegate::OnDecodedVideoFrame, - weak_factory_.GetWeakPtr()); -} - -void CastReceiverSessionDelegate::ReceivePacket( - std::unique_ptr<media::cast::Packet> packet) { - cast_receiver_->ReceivePacket(std::move(packet)); -} - -void CastReceiverSessionDelegate::StartAudio( - scoped_refptr<CastReceiverAudioValve> audio_valve) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - audio_valve_ = audio_valve; - cast_receiver_->RequestDecodedAudioFrame(on_audio_decoded_cb_); -} - -void CastReceiverSessionDelegate::OnDecodedAudioFrame( - std::unique_ptr<media::AudioBus> audio_bus, - base::TimeTicks playout_time, - bool is_continuous) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - if (!audio_valve_) - return; - - // We're on the IO thread, which doesn't allow blocking - // operations. Since we don't know what the Capture callback - // will do exactly, we need to jump to a different thread. - // Let's re-use the audio decoder thread. - cast_environment_->PostTask( - media::cast::CastEnvironment::AUDIO, FROM_HERE, - base::BindOnce(&CastReceiverAudioValve::DeliverDecodedAudio, audio_valve_, - base::Owned(audio_bus.release()), playout_time)); - cast_receiver_->RequestDecodedAudioFrame(on_audio_decoded_cb_); -} - -void CastReceiverSessionDelegate::StartVideo( - blink::VideoCaptureDeliverFrameCB video_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - frame_callback_ = video_callback; - cast_receiver_->RequestDecodedVideoFrame(on_video_decoded_cb_); -} - -void CastReceiverSessionDelegate::StopVideo() { - frame_callback_ = blink::VideoCaptureDeliverFrameCB(); -} - -void CastReceiverSessionDelegate::OnDecodedVideoFrame( - scoped_refptr<media::VideoFrame> video_frame, - base::TimeTicks playout_time, - bool is_continuous) { - if (frame_callback_.is_null()) - return; - frame_callback_.Run(std::move(video_frame), playout_time); - cast_receiver_->RequestDecodedVideoFrame(on_video_decoded_cb_); -} diff --git a/chromium/chrome/renderer/media/cast_receiver_session_delegate.h b/chromium/chrome/renderer/media/cast_receiver_session_delegate.h deleted file mode 100644 index e7708cb3e2f..00000000000 --- a/chromium/chrome/renderer/media/cast_receiver_session_delegate.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2015 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_DELEGATE_H_ -#define CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_DELEGATE_H_ - -#include <memory> - -#include "base/macros.h" -#include "chrome/renderer/media/cast_receiver_audio_valve.h" -#include "chrome/renderer/media/cast_session_delegate.h" -#include "media/capture/video_capture_types.h" -#include "media/cast/cast_receiver.h" -#include "third_party/blink/public/common/media/video_capture.h" - -class CastReceiverSessionDelegate : public CastSessionDelegateBase { - public: - typedef base::Callback<void(const std::string&)> ErrorCallback; - - CastReceiverSessionDelegate(); - ~CastReceiverSessionDelegate() override; - - void ReceivePacket(std::unique_ptr<media::cast::Packet> packet) override; - - void Start(const media::cast::FrameReceiverConfig& audio_config, - const media::cast::FrameReceiverConfig& video_config, - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const media::VideoCaptureFormat& format, - const ErrorCallback& error_callback); - - void StartAudio(scoped_refptr<CastReceiverAudioValve> audio_valve); - - void StartVideo(blink::VideoCaptureDeliverFrameCB frame_callback); - // Stop Video callbacks (eventually). - void StopVideo(); - - private: - void OnDecodedAudioFrame(std::unique_ptr<media::AudioBus> audio_bus, - base::TimeTicks playout_time, - bool is_continuous); - - void OnDecodedVideoFrame(scoped_refptr<media::VideoFrame> video_frame, - base::TimeTicks playout_time, - bool is_continuous); - - scoped_refptr<CastReceiverAudioValve> audio_valve_; - blink::VideoCaptureDeliverFrameCB frame_callback_; - media::cast::AudioFrameDecodedCallback on_audio_decoded_cb_; - media::cast::VideoFrameDecodedCallback on_video_decoded_cb_; - std::unique_ptr<media::cast::CastReceiver> cast_receiver_; - media::VideoCaptureFormat format_; - base::WeakPtrFactory<CastReceiverSessionDelegate> weak_factory_{this}; - DISALLOW_COPY_AND_ASSIGN(CastReceiverSessionDelegate); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RECEIVER_SESSION_DELEGATE_H_ diff --git a/chromium/chrome/renderer/media/cast_rtp_stream.cc b/chromium/chrome/renderer/media/cast_rtp_stream.cc deleted file mode 100644 index 1deb6336bbd..00000000000 --- a/chromium/chrome/renderer/media/cast_rtp_stream.cc +++ /dev/null @@ -1,578 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_rtp_stream.h" - -#include <stdint.h> - -#include <algorithm> -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/bind_helpers.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/timer/timer.h" -#include "base/trace_event/trace_event.h" -#include "base/values.h" -#include "chrome/common/chrome_switches.h" -#include "chrome/renderer/media/cast_session.h" -#include "chrome/renderer/media/cast_udp_transport.h" -#include "content/public/renderer/render_thread.h" -#include "content/public/renderer/video_encode_accelerator.h" -#include "media/base/audio_bus.h" -#include "media/base/audio_converter.h" -#include "media/base/audio_parameters.h" -#include "media/base/bind_to_current_loop.h" -#include "media/base/limits.h" -#include "media/base/video_frame.h" -#include "media/base/video_util.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_sender.h" -#include "media/cast/net/cast_transport_config.h" -#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_audio_sink.h" -#include "third_party/blink/public/platform/modules/mediastream/web_media_stream_sink.h" -#include "third_party/blink/public/platform/web_media_stream_source.h" -#include "third_party/blink/public/web/modules/mediastream/web_media_stream_utils.h" -#include "ui/gfx/geometry/size.h" - -using media::cast::FrameSenderConfig; - -namespace { - -// The maximum number of milliseconds that should elapse since the last video -// frame was received from the video source, before requesting refresh frames. -const int kRefreshIntervalMilliseconds = 250; - -// The maximum number of refresh video frames to request/receive. After this -// limit (60 * 250ms = 15 seconds), refresh frame requests will stop being made. -const int kMaxConsecutiveRefreshFrames = 60; - -FrameSenderConfig DefaultOpusConfig() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::AUDIO_OPUS; - config.sender_ssrc = 1; - config.receiver_ssrc = 2; - config.rtp_timebase = media::cast::kDefaultAudioSamplingRate; - config.channels = 2; - config.min_bitrate = config.max_bitrate = config.start_bitrate = - media::cast::kDefaultAudioEncoderBitrate; - config.max_frame_rate = 100; // 10 ms audio frames - config.codec = media::cast::CODEC_AUDIO_OPUS; - return config; -} - -FrameSenderConfig DefaultVp8Config() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::VIDEO_VP8; - config.sender_ssrc = 11; - config.receiver_ssrc = 12; - config.rtp_timebase = media::cast::kVideoFrequency; - config.channels = 1; - config.max_bitrate = media::cast::kDefaultMaxVideoBitrate; - config.min_bitrate = media::cast::kDefaultMinVideoBitrate; - config.max_frame_rate = media::cast::kDefaultMaxFrameRate; - config.codec = media::cast::CODEC_VIDEO_VP8; - return config; -} - -FrameSenderConfig DefaultH264Config() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::VIDEO_H264; - config.sender_ssrc = 11; - config.receiver_ssrc = 12; - config.rtp_timebase = media::cast::kVideoFrequency; - config.channels = 1; - config.max_bitrate = media::cast::kDefaultMaxVideoBitrate; - config.min_bitrate = media::cast::kDefaultMinVideoBitrate; - config.max_frame_rate = media::cast::kDefaultMaxFrameRate; - config.codec = media::cast::CODEC_VIDEO_H264; - return config; -} - -FrameSenderConfig DefaultRemotingAudioConfig() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::REMOTE_AUDIO; - config.sender_ssrc = 3; - config.receiver_ssrc = 4; - config.codec = media::cast::CODEC_AUDIO_REMOTE; - config.rtp_timebase = media::cast::kRemotingRtpTimebase; - config.max_bitrate = 1000000; - config.min_bitrate = 0; - config.channels = 2; - config.max_frame_rate = 100; // 10 ms audio frames - - return config; -} - -FrameSenderConfig DefaultRemotingVideoConfig() { - FrameSenderConfig config; - config.rtp_payload_type = media::cast::RtpPayloadType::REMOTE_VIDEO; - config.sender_ssrc = 13; - config.receiver_ssrc = 14; - config.codec = media::cast::CODEC_VIDEO_REMOTE; - config.rtp_timebase = media::cast::kRemotingRtpTimebase; - config.max_bitrate = 10000000; - config.min_bitrate = 0; - config.channels = 1; - config.max_frame_rate = media::cast::kDefaultMaxFrameRate; - return config; -} - -std::vector<FrameSenderConfig> SupportedAudioConfigs(bool for_remoting_stream) { - if (for_remoting_stream) - return {DefaultRemotingAudioConfig()}; - else - return {DefaultOpusConfig()}; -} - -std::vector<FrameSenderConfig> SupportedVideoConfigs(bool for_remoting_stream) { - if (for_remoting_stream) - return {DefaultRemotingVideoConfig()}; - - std::vector<FrameSenderConfig> supported_configs; - // Prefer VP8 over H.264 for hardware encoder. - if (CastRtpStream::IsHardwareVP8EncodingSupported()) - supported_configs.push_back(DefaultVp8Config()); - if (CastRtpStream::IsHardwareH264EncodingSupported()) - supported_configs.push_back(DefaultH264Config()); - - // Propose the default software VP8 encoder, if no hardware encoders are - // available. - if (supported_configs.empty()) - supported_configs.push_back(DefaultVp8Config()); - - return supported_configs; -} - -} // namespace - -// This class receives MediaStreamTrack events and video frames from a -// MediaStreamVideoTrack. It also includes a timer to request refresh frames -// when the capturer halts (e.g., a screen capturer stops delivering frames -// because the screen is not being updated). When a halt is detected, refresh -// frames will be requested at regular intervals for a short period of time. -// This provides the video encoder, downstream, several copies of the last frame -// so that it may clear up lossy encoding artifacts. -// -// Threading: Video frames are received on the IO thread and then -// forwarded to media::cast::VideoFrameInput. The inner class, Deliverer, -// handles this. Otherwise, all methods and member variables of the outer class -// must only be accessed on the render thread. -class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>, - public blink::WebMediaStreamSink { - public: - // |track| provides data for this sink. - // |error_callback| is called if video formats don't match. - CastVideoSink(const blink::WebMediaStreamTrack& track, - CastRtpStream::ErrorCallback error_callback) - : track_(track), - deliverer_(base::MakeRefCounted<Deliverer>(std::move(error_callback))), - consecutive_refresh_count_(0), - expecting_a_refresh_frame_(false), - is_connected_to_track_(false) {} - - ~CastVideoSink() override { - if (is_connected_to_track_) - blink::RemoveSinkFromMediaStreamTrack(track_, this); - } - - // Attach this sink to a video track represented by |track_|. - // Data received from the track will be submitted to |frame_input|. - void AddToTrack( - bool is_sink_secure, - const scoped_refptr<media::cast::VideoFrameInput>& frame_input) { - DCHECK(deliverer_); - deliverer_->WillConnectToTrack(AsWeakPtr(), frame_input); - refresh_timer_.Start( - FROM_HERE, - base::TimeDelta::FromMilliseconds(kRefreshIntervalMilliseconds), - base::Bind(&CastVideoSink::OnRefreshTimerFired, - base::Unretained(this))); - blink::AddSinkToMediaStreamTrack( - track_, this, base::BindRepeating(&Deliverer::OnVideoFrame, deliverer_), - is_sink_secure); - is_connected_to_track_ = true; - } - - private: - class Deliverer : public base::RefCountedThreadSafe<Deliverer> { - public: - explicit Deliverer(CastRtpStream::ErrorCallback error_callback) - : main_task_runner_(base::ThreadTaskRunnerHandle::Get()), - error_callback_(std::move(error_callback)) {} - - void WillConnectToTrack( - base::WeakPtr<CastVideoSink> sink, - scoped_refptr<media::cast::VideoFrameInput> frame_input) { - DCHECK(main_task_runner_->RunsTasksInCurrentSequence()); - sink_ = sink; - frame_input_ = std::move(frame_input); - } - - void OnVideoFrame(scoped_refptr<media::VideoFrame> video_frame, - base::TimeTicks estimated_capture_time) { - main_task_runner_->PostTask( - FROM_HERE, base::BindOnce(&CastVideoSink::DidReceiveFrame, sink_)); - - const base::TimeTicks timestamp = estimated_capture_time.is_null() - ? base::TimeTicks::Now() - : estimated_capture_time; - - if (!(video_frame->format() == media::PIXEL_FORMAT_I420 || - video_frame->format() == media::PIXEL_FORMAT_YV12 || - video_frame->format() == media::PIXEL_FORMAT_I420A)) { - error_callback_.Run("Incompatible video frame format."); - return; - } - scoped_refptr<media::VideoFrame> frame = video_frame; - // Drop alpha channel since we do not support it yet. - if (frame->format() == media::PIXEL_FORMAT_I420A) - frame = media::WrapAsI420VideoFrame(std::move(video_frame)); - - // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc - TRACE_EVENT_INSTANT2("cast_perf_test", "ConsumeVideoFrame", - TRACE_EVENT_SCOPE_THREAD, "timestamp", - (timestamp - base::TimeTicks()).InMicroseconds(), - "time_delta", frame->timestamp().InMicroseconds()); - frame_input_->InsertRawVideoFrame(std::move(frame), timestamp); - } - - private: - friend class base::RefCountedThreadSafe<Deliverer>; - ~Deliverer() {} - - const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_; - const CastRtpStream::ErrorCallback error_callback_; - - // These are set on the main thread after construction, and before the first - // call to OnVideoFrame() on the IO thread. |sink_| may be passed around on - // any thread, but must only be dereferenced on the main renderer thread. - base::WeakPtr<CastVideoSink> sink_; - scoped_refptr<media::cast::VideoFrameInput> frame_input_; - - DISALLOW_COPY_AND_ASSIGN(Deliverer); - }; - - private: - void OnRefreshTimerFired() { - ++consecutive_refresh_count_; - if (consecutive_refresh_count_ >= kMaxConsecutiveRefreshFrames) - refresh_timer_.Stop(); // Stop timer until receiving a non-refresh frame. - - DVLOG(1) << "CastVideoSink is requesting another refresh frame " - "(consecutive count=" << consecutive_refresh_count_ << ")."; - expecting_a_refresh_frame_ = true; - blink::RequestRefreshFrameFromVideoTrack(track_); - } - - void DidReceiveFrame() { - if (expecting_a_refresh_frame_) { - // There is uncertainty as to whether the video frame was in response to a - // refresh request. However, if it was not, more video frames will soon - // follow, and before the refresh timer can fire again. Thus, the - // behavior resulting from this logic will be correct. - expecting_a_refresh_frame_ = false; - } else { - consecutive_refresh_count_ = 0; - // The following re-starts the timer, scheduling it to fire at - // kRefreshIntervalMilliseconds from now. - refresh_timer_.Reset(); - } - } - - const blink::WebMediaStreamTrack track_; - const scoped_refptr<Deliverer> deliverer_; - - // Requests refresh frames at a constant rate while the source is paused, up - // to a consecutive maximum. - base::RepeatingTimer refresh_timer_; - - // Counter for the number of consecutive "refresh frames" requested. - int consecutive_refresh_count_; - - // Set to true when a request for a refresh frame has been made. This is - // cleared once the next frame is received. - bool expecting_a_refresh_frame_; - - bool is_connected_to_track_; - - DISALLOW_COPY_AND_ASSIGN(CastVideoSink); -}; - -// Receives audio data from a MediaStreamTrack. Data is submitted to -// media::cast::FrameInput. -// -// Threading: Audio frames are received on the real-time audio thread. -// Note that RemoveFromAudioTrack() is synchronous and we have -// gurantee that there will be no more audio data after calling it. -class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>, - public blink::WebMediaStreamAudioSink, - public media::AudioConverter::InputCallback { - public: - // |track| provides data for this sink. - CastAudioSink(const blink::WebMediaStreamTrack& track, - int output_channels, - int output_sample_rate) - : track_(track), - output_channels_(output_channels), - output_sample_rate_(output_sample_rate), - current_input_bus_(nullptr), - sample_frames_in_(0), - sample_frames_out_(0) {} - - ~CastAudioSink() override { - if (frame_input_.get()) - RemoveFromAudioTrack(this, track_); - } - - // Add this sink to the track. Data received from the track will be - // submitted to |frame_input|. - void AddToTrack( - const scoped_refptr<media::cast::AudioFrameInput>& frame_input) { - DCHECK(frame_input.get()); - DCHECK(!frame_input_.get()); - // This member is written here and then accessed on the IO thread - // We will not get data until AddToAudioTrack is called so it is - // safe to access this member now. - frame_input_ = frame_input; - AddToAudioTrack(this, track_); - } - - protected: - // Called on real-time audio thread. - void OnData(const media::AudioBus& input_bus, - base::TimeTicks estimated_capture_time) override { - DCHECK(input_params_.IsValid()); - DCHECK_EQ(input_bus.channels(), input_params_.channels()); - DCHECK_EQ(input_bus.frames(), input_params_.frames_per_buffer()); - DCHECK(!estimated_capture_time.is_null()); - DCHECK(converter_.get()); - - // Determine the duration of the audio signal enqueued within |converter_|. - const base::TimeDelta signal_duration_already_buffered = - (sample_frames_in_ * base::TimeDelta::FromSeconds(1) / - input_params_.sample_rate()) - - (sample_frames_out_ * base::TimeDelta::FromSeconds(1) / - output_sample_rate_); - DVLOG(2) << "Audio reference time adjustment: -(" - << signal_duration_already_buffered.InMicroseconds() << " us)"; - const base::TimeTicks capture_time_of_first_converted_sample = - estimated_capture_time - signal_duration_already_buffered; - - // Convert the entire input signal. AudioConverter is efficient in that no - // additional copying or conversion will occur if the input signal is in the - // same format as the output. Note that, while the number of sample frames - // provided as input is always the same, the chunk size (and the size of the - // |audio_bus| here) can be variable. This is not an issue since - // media::cast::AudioFrameInput can handle variable-sized AudioBuses. - std::unique_ptr<media::AudioBus> audio_bus = - media::AudioBus::Create(output_channels_, converter_->ChunkSize()); - // AudioConverter will call ProvideInput() to fetch from |current_data_|. - current_input_bus_ = &input_bus; - converter_->Convert(audio_bus.get()); - DCHECK(!current_input_bus_); // ProvideInput() called exactly once? - - sample_frames_in_ += input_params_.frames_per_buffer(); - sample_frames_out_ += audio_bus->frames(); - - frame_input_->InsertAudio(std::move(audio_bus), - capture_time_of_first_converted_sample); - } - - // Called on real-time audio thread. - void OnSetFormat(const media::AudioParameters& params) override { - if (input_params_.Equals(params)) - return; - input_params_ = params; - - DVLOG(1) << "Setting up audio resampling: {" - << input_params_.channels() << " channels, " - << input_params_.sample_rate() << " Hz} --> {" - << output_channels_ << " channels, " - << output_sample_rate_ << " Hz}"; - const media::AudioParameters output_params( - media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::GuessChannelLayout(output_channels_), output_sample_rate_, - output_sample_rate_ * input_params_.frames_per_buffer() / - input_params_.sample_rate()); - converter_.reset( - new media::AudioConverter(input_params_, output_params, false)); - converter_->AddInput(this); - sample_frames_in_ = 0; - sample_frames_out_ = 0; - } - - // Called on real-time audio thread. - double ProvideInput(media::AudioBus* audio_bus, - uint32_t frames_delayed) override { - DCHECK(current_input_bus_); - current_input_bus_->CopyTo(audio_bus); - current_input_bus_ = nullptr; - return 1.0; - } - - private: - const blink::WebMediaStreamTrack track_; - const int output_channels_; - const int output_sample_rate_; - - // This must be set before the real-time audio thread starts calling OnData(), - // and remain unchanged until after the thread will stop calling OnData(). - scoped_refptr<media::cast::AudioFrameInput> frame_input_; - - // These members are accessed on the real-time audio time only. - media::AudioParameters input_params_; - std::unique_ptr<media::AudioConverter> converter_; - const media::AudioBus* current_input_bus_; - int64_t sample_frames_in_; - int64_t sample_frames_out_; - - DISALLOW_COPY_AND_ASSIGN(CastAudioSink); -}; - -bool CastRtpStream::IsHardwareVP8EncodingSupported() { - // Query for hardware VP8 encoder support. - const std::vector<media::VideoEncodeAccelerator::SupportedProfile> - vea_profiles = content::GetSupportedVideoEncodeAcceleratorProfiles(); - for (const auto& vea_profile : vea_profiles) { - if (vea_profile.profile >= media::VP8PROFILE_MIN && - vea_profile.profile <= media::VP8PROFILE_MAX) { - return true; - } - } - return false; -} - -bool CastRtpStream::IsHardwareH264EncodingSupported() { -// Query for hardware H.264 encoder support. -// -// TODO(miu): Look into why H.264 hardware encoder on MacOS is broken. -// http://crbug.com/596674 -// TODO(emircan): Look into HW encoder initialization issues on Win. -// https://crbug.com/636064 -#if !defined(OS_MACOSX) && !defined(OS_WIN) - const std::vector<media::VideoEncodeAccelerator::SupportedProfile> - vea_profiles = content::GetSupportedVideoEncodeAcceleratorProfiles(); - for (const auto& vea_profile : vea_profiles) { - if (vea_profile.profile >= media::H264PROFILE_MIN && - vea_profile.profile <= media::H264PROFILE_MAX) { - return true; - } - } -#endif // !defined(OS_MACOSX) && !defined(OS_WIN) - return false; -} - -CastRtpStream::CastRtpStream(const blink::WebMediaStreamTrack& track, - const scoped_refptr<CastSession>& session) - : track_(track), - cast_session_(session), - is_audio_(track_.Source().GetType() == - blink::WebMediaStreamSource::kTypeAudio) {} - -CastRtpStream::CastRtpStream(bool is_audio, - const scoped_refptr<CastSession>& session) - : cast_session_(session), is_audio_(is_audio) {} - -CastRtpStream::~CastRtpStream() { - Stop(); -} - -std::vector<FrameSenderConfig> CastRtpStream::GetSupportedConfigs() { - if (is_audio_) - return SupportedAudioConfigs(track_.IsNull()); - else - return SupportedVideoConfigs(track_.IsNull()); -} - -void CastRtpStream::Start(int32_t stream_id, - const FrameSenderConfig& config, - base::OnceClosure start_callback, - base::OnceClosure stop_callback, - ErrorCallback error_callback) { - DCHECK(!start_callback.is_null()); - DCHECK(!stop_callback.is_null()); - DCHECK(!error_callback.is_null()); - - DVLOG(1) << "CastRtpStream::Start = " << (is_audio_ ? "audio" : "video"); - stop_callback_ = std::move(stop_callback); - error_callback_ = std::move(error_callback); - - if (track_.IsNull()) { - cast_session_->StartRemotingStream( - stream_id, config, - base::BindOnce(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())); - } else if (is_audio_) { - // In case of error we have to go through DidEncounterError() to stop - // the streaming after reporting the error. - audio_sink_.reset( - new CastAudioSink(track_, config.channels, config.rtp_timebase)); - cast_session_->StartAudio( - config, - base::Bind(&CastAudioSink::AddToTrack, audio_sink_->AsWeakPtr()), - base::Bind(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())); - } else { - // See the code for audio above for explanation of callbacks. - video_sink_.reset(new CastVideoSink( - track_, - media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())))); - cast_session_->StartVideo( - config, base::Bind(&CastVideoSink::AddToTrack, video_sink_->AsWeakPtr(), - !config.aes_key.empty()), - base::Bind(&CastRtpStream::DidEncounterError, - weak_factory_.GetWeakPtr())); - } - std::move(start_callback).Run(); -} - -void CastRtpStream::Stop() { - DVLOG(1) << "CastRtpStream::Stop = " << (is_audio_ ? "audio" : "video"); - if (stop_callback_.is_null()) - return; // Already stopped. - weak_factory_.InvalidateWeakPtrs(); - error_callback_.Reset(); - audio_sink_.reset(); - video_sink_.reset(); - std::move(stop_callback_).Run(); -} - -void CastRtpStream::ToggleLogging(bool enable) { - DVLOG(1) << "CastRtpStream::ToggleLogging(" << enable - << ") = " << (is_audio_ ? "audio" : "video"); - cast_session_->ToggleLogging(is_audio_, enable); -} - -void CastRtpStream::GetRawEvents( - const base::Callback<void(std::unique_ptr<base::Value>)>& callback, - const std::string& extra_data) { - DVLOG(1) << "CastRtpStream::GetRawEvents = " - << (is_audio_ ? "audio" : "video"); - cast_session_->GetEventLogsAndReset(is_audio_, extra_data, callback); -} - -void CastRtpStream::GetStats( - const base::Callback<void(std::unique_ptr<base::DictionaryValue>)>& - callback) { - DVLOG(1) << "CastRtpStream::GetStats = " << (is_audio_ ? "audio" : "video"); - cast_session_->GetStatsAndReset(is_audio_, callback); -} - -void CastRtpStream::DidEncounterError(const std::string& message) { - DCHECK(content::RenderThread::Get()); - DVLOG(1) << "CastRtpStream::DidEncounterError(" << message - << ") = " << (is_audio_ ? "audio" : "video"); - // Save the WeakPtr first because the error callback might delete this object. - base::WeakPtr<CastRtpStream> ptr = weak_factory_.GetWeakPtr(); - error_callback_.Run(message); - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(&CastRtpStream::Stop, ptr)); -} diff --git a/chromium/chrome/renderer/media/cast_rtp_stream.h b/chromium/chrome/renderer/media/cast_rtp_stream.h deleted file mode 100644 index f39eb55f724..00000000000 --- a/chromium/chrome/renderer/media/cast_rtp_stream.h +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_RTP_STREAM_H_ -#define CHROME_RENDERER_MEDIA_CAST_RTP_STREAM_H_ - -#include <memory> -#include <string> -#include <vector> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "media/cast/cast_config.h" -#include "media/cast/constants.h" -#include "third_party/blink/public/platform/web_media_stream_track.h" - -namespace base { -class DictionaryValue; -class Value; -} - -class CastAudioSink; -class CastSession; -class CastVideoSink; - -// This object represents a RTP stream that encodes and optionally -// encrypt audio or video data from a WebMediaStreamTrack. -// Note that this object does not actually output packets. It allows -// configuration of encoding and RTP parameters and control such a logical -// stream. -class CastRtpStream { - public: - using ErrorCallback = base::RepeatingCallback<void(const std::string&)>; - - static bool IsHardwareVP8EncodingSupported(); - - static bool IsHardwareH264EncodingSupported(); - - CastRtpStream(const blink::WebMediaStreamTrack& track, - const scoped_refptr<CastSession>& session); - CastRtpStream(bool is_audio, const scoped_refptr<CastSession>& session); - ~CastRtpStream(); - - // Return parameters currently supported by this stream. - std::vector<media::cast::FrameSenderConfig> GetSupportedConfigs(); - - // Begin encoding of media stream and then submit the encoded streams - // to underlying transport. - // |stream_id| is the unique ID of this stream. - // When the stream is started |start_callback| is called. - // When the stream is stopped |stop_callback| is called. - // When there is an error |error_callback| is called with a message. - void Start(int32_t stream_id, - const media::cast::FrameSenderConfig& config, - base::OnceClosure start_callback, - base::OnceClosure stop_callback, - ErrorCallback error_callback); - - // Stop encoding. - void Stop(); - - // Enables or disables logging for this stream. - void ToggleLogging(bool enable); - - // Get serialized raw events for this stream with |extra_data| attached, - // and invokes |callback| with the result. - void GetRawEvents( - const base::Callback<void(std::unique_ptr<base::Value>)>& callback, - const std::string& extra_data); - - // Get stats in DictionaryValue format and invokves |callback| with - // the result. - void GetStats(const base::Callback< - void(std::unique_ptr<base::DictionaryValue>)>& callback); - - private: - void DidEncounterError(const std::string& message); - - blink::WebMediaStreamTrack track_; - const scoped_refptr<CastSession> cast_session_; - std::unique_ptr<CastAudioSink> audio_sink_; - std::unique_ptr<CastVideoSink> video_sink_; - base::OnceClosure stop_callback_; - ErrorCallback error_callback_; - bool is_audio_; - - base::WeakPtrFactory<CastRtpStream> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastRtpStream); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_RTP_STREAM_H_ diff --git a/chromium/chrome/renderer/media/cast_session.cc b/chromium/chrome/renderer/media/cast_session.cc deleted file mode 100644 index fd031a658a3..00000000000 --- a/chromium/chrome/renderer/media/cast_session.cc +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_session.h" - -#include <stddef.h> - -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/memory/unsafe_shared_memory_region.h" -#include "base/single_thread_task_runner.h" -#include "chrome/renderer/media/cast_session_delegate.h" -#include "content/public/renderer/render_thread.h" -#include "content/public/renderer/video_encode_accelerator.h" -#include "media/base/bind_to_current_loop.h" -#include "media/base/video_frame.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/logging_defines.h" - -namespace { - -void CreateVideoEncodeAccelerator( - const media::cast::ReceiveVideoEncodeAcceleratorCallback& callback) { - DCHECK(content::RenderThread::Get()); - - // Delegate the call to content API on the render thread. - content::CreateVideoEncodeAccelerator(callback); -} - -void CreateVideoEncodeMemory( - size_t size, - const media::cast::ReceiveVideoEncodeMemoryCallback& callback) { - DCHECK(content::RenderThread::Get()); - - base::UnsafeSharedMemoryRegion shm = - base::UnsafeSharedMemoryRegion::Create(size); - DCHECK(shm.IsValid()) << "Failed to allocate shared memory"; - callback.Run(std::move(shm)); -} - -} // namespace - -CastSession::CastSession( - scoped_refptr<base::SingleThreadTaskRunner> task_runner) - : delegate_(new CastSessionDelegate()), - main_thread_task_runner_(std::move(task_runner)), - io_task_runner_(content::RenderThread::Get()->GetIOTaskRunner()) {} - -CastSession::~CastSession() { - // We should always be able to delete the object on the IO thread. - CHECK(io_task_runner_->DeleteSoon(FROM_HERE, delegate_.release())); -} - -void CastSession::StartAudio(const media::cast::FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback) { - DCHECK(content::RenderThread::Get()); - - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce( - &CastSessionDelegate::StartAudio, base::Unretained(delegate_.get()), - config, media::BindToLoop(main_thread_task_runner_, callback), - media::BindToLoop(main_thread_task_runner_, error_callback))); -} - -void CastSession::StartVideo(const media::cast::FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback) { - DCHECK(content::RenderThread::Get()); - - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce( - &CastSessionDelegate::StartVideo, base::Unretained(delegate_.get()), - config, media::BindToLoop(main_thread_task_runner_, callback), - media::BindToLoop(main_thread_task_runner_, error_callback), - media::BindToLoop(main_thread_task_runner_, - base::Bind(&CreateVideoEncodeAccelerator)), - media::BindToLoop(main_thread_task_runner_, - base::Bind(&CreateVideoEncodeMemory)))); -} - -void CastSession::StartRemotingStream( - int32_t stream_id, - const media::cast::FrameSenderConfig& config, - ErrorOnceCallback error_callback) { - DCHECK(content::RenderThread::Get()); - - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::StartRemotingStream, - base::Unretained(delegate_.get()), stream_id, config, - media::BindToLoop(main_thread_task_runner_, - std::move(error_callback)))); -} - -void CastSession::StartUDP(const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce( - &CastSessionDelegate::StartUDP, base::Unretained(delegate_.get()), - net::IPEndPoint(), remote_endpoint, std::move(options), - media::BindToLoop(main_thread_task_runner_, error_callback))); -} - -void CastSession::ToggleLogging(bool is_audio, bool enable) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::ToggleLogging, - base::Unretained(delegate_.get()), is_audio, enable)); -} - -void CastSession::GetEventLogsAndReset( - bool is_audio, const std::string& extra_data, - const EventLogsCallback& callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::GetEventLogsAndReset, - base::Unretained(delegate_.get()), is_audio, extra_data, - media::BindToLoop(main_thread_task_runner_, callback))); -} - -void CastSession::GetStatsAndReset(bool is_audio, - const StatsCallback& callback) { - io_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&CastSessionDelegate::GetStatsAndReset, - base::Unretained(delegate_.get()), is_audio, - media::BindToLoop(main_thread_task_runner_, callback))); -} diff --git a/chromium/chrome/renderer/media/cast_session.h b/chromium/chrome/renderer/media/cast_session.h deleted file mode 100644 index 6a3f79e04fd..00000000000 --- a/chromium/chrome/renderer/media/cast_session.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_SESSION_H_ -#define CHROME_RENDERER_MEDIA_CAST_SESSION_H_ - -#include <memory> -#include <vector> - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "media/cast/cast_config.h" -#include "net/base/ip_endpoint.h" - -namespace base { -class DictionaryValue; -class SingleThreadTaskRunner; -class Value; -} // namespace base - -namespace media { -namespace cast { -class AudioFrameInput; -class VideoFrameInput; -} // namespace cast -} // namespace media - -class CastSessionDelegate; - -// This class represents a Cast session and allows the session to be -// configured on the main thread. Actual work is forwarded to -// CastSessionDelegate on the IO thread. -class CastSession : public base::RefCounted<CastSession> { - public: - using AudioFrameInputAvailableCallback = - base::Callback<void(const scoped_refptr<media::cast::AudioFrameInput>&)>; - using VideoFrameInputAvailableCallback = - base::Callback<void(const scoped_refptr<media::cast::VideoFrameInput>&)>; - using SendPacketCallback = base::Callback<void(const std::vector<char>&)>; - using EventLogsCallback = base::Callback<void(std::unique_ptr<base::Value>)>; - using StatsCallback = - base::Callback<void(std::unique_ptr<base::DictionaryValue>)>; - // TODO(crbug.com/1007641): remove ErrorCallback and rename ErrorOnceCallback - // once all occurrences of base::Callback have been removed. - using ErrorCallback = base::Callback<void(const std::string&)>; - using ErrorOnceCallback = base::OnceCallback<void(const std::string&)>; - - explicit CastSession(scoped_refptr<base::SingleThreadTaskRunner> task_runner); - - // Start encoding of audio and video using the provided configuration. - // - // When Cast sender is started and ready to be used - // media::cast::FrameInput will be given through |callback|. - // If it encounters an error, |error_callback| will be invoked with the - // error message. Both |callback| and |error_callback| will be made on - // the main thread. - // |StartUDP()| must be called before these methods. - void StartAudio(const media::cast::FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback); - void StartVideo(const media::cast::FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback); - - // Start remoting a stream. |error_callback| will be invoked when any error - // occurs. |StartUDP()| must be called before calling this method. - void StartRemotingStream(int32_t stream_id, - const media::cast::FrameSenderConfig& config, - ErrorOnceCallback error_callback); - - // This will create the Cast transport and connect to |remote_endpoint|. - // |options| is a dictionary which contain optional configuration for the - // udp transport. - // Must be called before initialization of audio or video. - void StartUDP(const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback); - - // Creates or destroys event subscriber for the audio or video stream. - // |is_audio|: true if the event subscriber is for audio. Video otherwise. - // |enable|: If true, creates an event subscriber. Otherwise destroys - // existing subscriber and discards logs. - void ToggleLogging(bool is_audio, bool enable); - - // Returns raw event logs in serialized format for either the audio or video - // stream since last call and returns result in |callback|. Also attaches - // |extra_data| to the log. - void GetEventLogsAndReset(bool is_audio, - const std::string& extra_data, const EventLogsCallback& callback); - - // Returns stats in a DictionaryValue format for either the audio or video - // stream since last call and returns result in |callback|. - void GetStatsAndReset(bool is_audio, const StatsCallback& callback); - - private: - friend class base::RefCounted<CastSession>; - virtual ~CastSession(); - - // This member should never be dereferenced on the main thread. - // CastSessionDelegate lives only on the IO thread. It is always - // safe to post task on the IO thread to access CastSessionDelegate - // because it is owned by this object. - std::unique_ptr<CastSessionDelegate> delegate_; - - // A main thread task runner that might execute JavaScript. - const scoped_refptr<base::SingleThreadTaskRunner> main_thread_task_runner_; - - // Proxy to the IO task runner. - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - - DISALLOW_COPY_AND_ASSIGN(CastSession); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_SESSION_H_ diff --git a/chromium/chrome/renderer/media/cast_session_browsertest.cc b/chromium/chrome/renderer/media/cast_session_browsertest.cc deleted file mode 100644 index c4c4337d7e8..00000000000 --- a/chromium/chrome/renderer/media/cast_session_browsertest.cc +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_session.h" - -#include "base/run_loop.h" -#include "base/threading/thread_task_runner_handle.h" -#include "chrome/renderer/chrome_content_renderer_client.h" -#include "chrome/test/base/chrome_render_view_test.h" -#include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h" - -typedef ChromeRenderViewTest CastSessionBrowserTest; - -// Tests that CastSession is created and destroyed properly inside -// chrome renderer. -TEST_F(CastSessionBrowserTest, CreateAndDestroy) { - chrome_render_thread_->set_io_task_runner( - blink::scheduler::GetSingleThreadTaskRunnerForTesting()); - ChromeContentRendererClient* client = - static_cast<ChromeContentRendererClient*>(content_renderer_client_.get()); - client->RenderThreadStarted(); - - scoped_refptr<CastSession> session( - new CastSession(blink::scheduler::GetSingleThreadTaskRunnerForTesting())); - - // Causes CastSession to destruct. - session.reset(); - base::RunLoop().RunUntilIdle(); -} diff --git a/chromium/chrome/renderer/media/cast_session_delegate.cc b/chromium/chrome/renderer/media/cast_session_delegate.cc deleted file mode 100644 index 7bc22f2ecc5..00000000000 --- a/chromium/chrome/renderer/media/cast_session_delegate.cc +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_session_delegate.h" - -#include <memory> -#include <utility> -#include <vector> - -#include "base/bind.h" -#include "base/lazy_instance.h" -#include "base/logging.h" -#include "base/single_thread_task_runner.h" -#include "base/strings/stringprintf.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/values.h" -#include "build/build_config.h" -#include "chrome/renderer/media/cast_threads.h" -#include "chrome/renderer/media/cast_transport_ipc.h" -#include "components/version_info/version_info.h" -#include "content/public/renderer/render_thread.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_environment.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/log_serializer.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/logging/proto/raw_events.pb.h" -#include "media/cast/logging/raw_event_subscriber_bundle.h" -#include "media/cast/net/cast_transport.h" -#include "media/cast/net/cast_transport_config.h" - -using media::cast::CastEnvironment; -using media::cast::CastSender; -using media::cast::FrameSenderConfig; - -static base::LazyInstance<CastThreads>::DestructorAtExit g_cast_threads = - LAZY_INSTANCE_INITIALIZER; - -CastSessionDelegateBase::CastSessionDelegateBase() - : io_task_runner_(content::RenderThread::Get()->GetIOTaskRunner()) { - DCHECK(io_task_runner_.get()); -#if defined(OS_WIN) - // Note that this also increases the accuracy of PostDelayTask, - // which is is very helpful to cast. - if (!base::Time::ActivateHighResolutionTimer(true)) { - LOG(WARNING) << "Failed to activate high resolution timers for cast."; - } -#endif -} - -CastSessionDelegateBase::~CastSessionDelegateBase() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); -#if defined(OS_WIN) - base::Time::ActivateHighResolutionTimer(false); -#endif -} - -void CastSessionDelegateBase::StartUDP( - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - // CastSender uses the renderer's IO thread as the main thread. This reduces - // thread hopping for incoming video frames and outgoing network packets. - // TODO(hubbe): Create cast environment in ctor instead. - cast_environment_ = - new CastEnvironment(base::DefaultTickClock::GetInstance(), - base::ThreadTaskRunnerHandle::Get(), - g_cast_threads.Get().GetAudioEncodeTaskRunner(), - g_cast_threads.Get().GetVideoEncodeTaskRunner()); - - // Rationale for using unretained: The callback cannot be called after the - // destruction of CastTransportIPC, and they both share the same thread. - cast_transport_ = std::make_unique<CastTransportIPC>( - local_endpoint, remote_endpoint, std::move(options), - base::BindRepeating(&CastSessionDelegateBase::ReceivePacket, - base::Unretained(this)), - base::BindRepeating(&CastSessionDelegateBase::StatusNotificationCB, - base::Unretained(this), error_callback), - base::BindRepeating( - &media::cast::LogEventDispatcher::DispatchBatchOfEvents, - base::Unretained(cast_environment_->logger()))); -} - -void CastSessionDelegateBase::StatusNotificationCB( - ErrorOnceCallback error_callback, - media::cast::CastTransportStatus status) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - std::string error_message; - - switch (status) { - case media::cast::TRANSPORT_STREAM_UNINITIALIZED: - case media::cast::TRANSPORT_STREAM_INITIALIZED: - return; // Not errors, do nothing. - case media::cast::TRANSPORT_INVALID_CRYPTO_CONFIG: - std::move(error_callback).Run("Invalid encrypt/decrypt configuration."); - break; - case media::cast::TRANSPORT_SOCKET_ERROR: - std::move(error_callback).Run("Socket error."); - break; - } -} - -CastSessionDelegate::CastSessionDelegate() { - DCHECK(io_task_runner_.get()); -} - -CastSessionDelegate::~CastSessionDelegate() { - DCHECK(io_task_runner_->BelongsToCurrentThread()); -} - -void CastSessionDelegate::StartAudio( - const FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - ErrorOnceCallback error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!cast_transport_ || !cast_sender_) { - std::move(error_callback).Run("Destination not set."); - return; - } - - audio_frame_input_available_callback_ = callback; - cast_sender_->InitializeAudio( - config, base::BindOnce(&CastSessionDelegate::OnOperationalStatusChange, - weak_factory_.GetWeakPtr(), true, - std::move(error_callback))); -} - -void CastSessionDelegate::StartVideo( - const FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback, - const media::cast::CreateVideoEncodeAcceleratorCallback& create_vea_cb, - const media::cast::CreateVideoEncodeMemoryCallback& - create_video_encode_mem_cb) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!cast_transport_ || !cast_sender_) { - error_callback.Run("Destination not set."); - return; - } - - video_frame_input_available_callback_ = callback; - - cast_sender_->InitializeVideo( - config, - base::BindRepeating(&CastSessionDelegate::OnOperationalStatusChange, - weak_factory_.GetWeakPtr(), false, error_callback), - create_vea_cb, create_video_encode_mem_cb); -} - -void CastSessionDelegate::StartRemotingStream( - int32_t stream_id, - const FrameSenderConfig& config, - ErrorOnceCallback error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!cast_transport_) { - std::move(error_callback).Run("Destination not set."); - return; - } - - media::cast::CastTransportRtpConfig transport_config; - transport_config.ssrc = config.sender_ssrc; - transport_config.feedback_ssrc = config.receiver_ssrc; - transport_config.rtp_payload_type = config.rtp_payload_type; - transport_config.rtp_stream_id = stream_id; - transport_config.aes_key = config.aes_key; - transport_config.aes_iv_mask = config.aes_iv_mask; - cast_transport_->InitializeStream(transport_config, nullptr); -} - -void CastSessionDelegate::StartUDP( - const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - CastSessionDelegateBase::StartUDP(local_endpoint, remote_endpoint, - std::move(options), error_callback); - event_subscribers_ = std::make_unique<media::cast::RawEventSubscriberBundle>( - cast_environment_); - - cast_sender_ = CastSender::Create(cast_environment_, cast_transport_.get()); -} - -void CastSessionDelegate::ToggleLogging(bool is_audio, bool enable) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - if (!event_subscribers_.get()) - return; - - if (enable) - event_subscribers_->AddEventSubscribers(is_audio); - else - event_subscribers_->RemoveEventSubscribers(is_audio); -} - -void CastSessionDelegate::GetEventLogsAndReset( - bool is_audio, - const std::string& extra_data, - const EventLogsCallback& callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!event_subscribers_.get()) { - callback.Run(std::make_unique<base::Value>(base::Value::Type::BINARY)); - return; - } - - media::cast::EncodingEventSubscriber* subscriber = - event_subscribers_->GetEncodingEventSubscriber(is_audio); - if (!subscriber) { - callback.Run(std::make_unique<base::Value>(base::Value::Type::BINARY)); - return; - } - - media::cast::proto::LogMetadata metadata; - media::cast::FrameEventList frame_events; - media::cast::PacketEventList packet_events; - - subscriber->GetEventsAndReset(&metadata, &frame_events, &packet_events); - - if (!extra_data.empty()) - metadata.set_extra_data(extra_data); - media::cast::proto::GeneralDescription* gen_desc = - metadata.mutable_general_description(); - gen_desc->set_product(version_info::GetProductName()); - gen_desc->set_product_version(version_info::GetVersionNumber()); - gen_desc->set_os(version_info::GetOSType()); - - std::unique_ptr<char[]> serialized_log( - new char[media::cast::kMaxSerializedBytes]); - int output_bytes; - bool success = media::cast::SerializeEvents(metadata, - frame_events, - packet_events, - true, - media::cast::kMaxSerializedBytes, - serialized_log.get(), - &output_bytes); - - if (!success) { - DVLOG(2) << "Failed to serialize event log."; - callback.Run(std::make_unique<base::Value>(base::Value::Type::BINARY)); - return; - } - - DVLOG(2) << "Serialized log length: " << output_bytes; - - auto blob = std::make_unique<base::Value>(std::vector<char>( - serialized_log.get(), serialized_log.get() + output_bytes)); - callback.Run(std::move(blob)); -} - -void CastSessionDelegate::GetStatsAndReset(bool is_audio, - const StatsCallback& callback) { - DCHECK(io_task_runner_->BelongsToCurrentThread()); - - if (!event_subscribers_.get()) { - callback.Run(std::make_unique<base::DictionaryValue>()); - return; - } - - media::cast::StatsEventSubscriber* subscriber = - event_subscribers_->GetStatsEventSubscriber(is_audio); - if (!subscriber) { - callback.Run(std::make_unique<base::DictionaryValue>()); - return; - } - - std::unique_ptr<base::DictionaryValue> stats = subscriber->GetStats(); - subscriber->Reset(); - - callback.Run(std::move(stats)); -} - -void CastSessionDelegate::OnOperationalStatusChange( - bool is_for_audio, - ErrorOnceCallback error_callback, - media::cast::OperationalStatus status) { - DCHECK(cast_sender_); - - switch (status) { - case media::cast::STATUS_UNINITIALIZED: - case media::cast::STATUS_CODEC_REINIT_PENDING: - // Not an error. - // TODO(miu): As an optimization, signal the client to pause sending more - // frames until the state becomes STATUS_INITIALIZED again. - break; - case media::cast::STATUS_INITIALIZED: - // Once initialized, run the "frame input available" callback to allow the - // client to begin sending frames. If STATUS_INITIALIZED is encountered - // again, do nothing since this is only an indication that the codec has - // successfully re-initialized. - if (is_for_audio) { - if (!audio_frame_input_available_callback_.is_null()) { - std::move(audio_frame_input_available_callback_) - .Run(cast_sender_->audio_frame_input()); - } - } else { - if (!video_frame_input_available_callback_.is_null()) { - std::move(video_frame_input_available_callback_) - .Run(cast_sender_->video_frame_input()); - } - } - break; - case media::cast::STATUS_INVALID_CONFIGURATION: - std::move(error_callback) - .Run(base::StringPrintf("Invalid %s configuration.", - is_for_audio ? "audio" : "video")); - break; - case media::cast::STATUS_UNSUPPORTED_CODEC: - std::move(error_callback) - .Run(base::StringPrintf("%s codec not supported.", - is_for_audio ? "Audio" : "Video")); - break; - case media::cast::STATUS_CODEC_INIT_FAILED: - std::move(error_callback) - .Run(base::StringPrintf("%s codec initialization failed.", - is_for_audio ? "Audio" : "Video")); - break; - case media::cast::STATUS_CODEC_RUNTIME_ERROR: - std::move(error_callback) - .Run(base::StringPrintf("%s codec runtime error.", - is_for_audio ? "Audio" : "Video")); - break; - } -} - -void CastSessionDelegate::ReceivePacket( - std::unique_ptr<media::cast::Packet> packet) { - // Do nothing (frees packet) -} diff --git a/chromium/chrome/renderer/media/cast_session_delegate.h b/chromium/chrome/renderer/media/cast_session_delegate.h deleted file mode 100644 index b841c35479c..00000000000 --- a/chromium/chrome/renderer/media/cast_session_delegate.h +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_SESSION_DELEGATE_H_ -#define CHROME_RENDERER_MEDIA_CAST_SESSION_DELEGATE_H_ - -#include <map> -#include <memory> -#include <vector> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "base/threading/thread.h" -#include "base/threading/thread_checker.h" -#include "base/time/default_tick_clock.h" -#include "media/cast/cast_config.h" -#include "media/cast/cast_sender.h" -#include "media/cast/logging/logging_defines.h" - -namespace base { -class DictionaryValue; -class SingleThreadTaskRunner; -class Value; -} // namespace base - -namespace media { - -namespace cast { -class CastEnvironment; -class RawEventSubscriberBundle; - -namespace transport { -class CastTransport; -} // namespace transport -} // namespace cast -} // namespace media - -// Breaks out functionality that is common between CastSessionDelegate and -// CastReceiverSessionDelegate. -class CastSessionDelegateBase { - public: - // TODO(crbug.com/1007641): remove ErrorCallback and rename ErrorOnceCallback - // once all occurrences of base::Callback have been removed. - using ErrorCallback = base::RepeatingCallback<void(const std::string&)>; - using ErrorOnceCallback = base::OnceCallback<void(const std::string&)>; - - CastSessionDelegateBase(); - virtual ~CastSessionDelegateBase(); - - // This will start the session by configuring and creating the Cast transport - // and the Cast sender. - // Must be called before initialization of audio or video. - void StartUDP(const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback); - - protected: - void StatusNotificationCB(ErrorOnceCallback error_callback, - media::cast::CastTransportStatus status); - - virtual void ReceivePacket(std::unique_ptr<media::cast::Packet> packet) = 0; - - base::ThreadChecker thread_checker_; - scoped_refptr<media::cast::CastEnvironment> cast_environment_; - std::unique_ptr<media::cast::CastTransport> cast_transport_; - - // Proxy to the IO message loop. - const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_; - base::WeakPtrFactory<CastSessionDelegateBase> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastSessionDelegateBase); -}; - -// This class hosts CastSender and connects it to audio/video frame input -// and network socket. -// This class is created on the render thread and destroyed on the IO -// thread. All methods are accessible only on the IO thread. -class CastSessionDelegate : public CastSessionDelegateBase { - public: - typedef base::Callback<void(const scoped_refptr< - media::cast::AudioFrameInput>&)> AudioFrameInputAvailableCallback; - typedef base::Callback<void(const scoped_refptr< - media::cast::VideoFrameInput>&)> VideoFrameInputAvailableCallback; - typedef base::Callback<void(std::unique_ptr<base::Value>)> EventLogsCallback; - typedef base::Callback<void(std::unique_ptr<base::DictionaryValue>)> - StatsCallback; - - CastSessionDelegate(); - ~CastSessionDelegate() override; - - void StartUDP(const net::IPEndPoint& local_endpoint, - const net::IPEndPoint& remote_endpoint, - std::unique_ptr<base::DictionaryValue> options, - const ErrorCallback& error_callback); - - // After calling StartAudio() or StartVideo() encoding of that media will - // begin as soon as data is delivered to its sink, if the second method is - // called the first media will be restarted. It is strongly recommended not to - // deliver any data between calling the two methods. - // It's OK to call only one of the two methods. - // StartUDP must be called before these methods. - void StartAudio(const media::cast::FrameSenderConfig& config, - const AudioFrameInputAvailableCallback& callback, - ErrorOnceCallback error_callback); - - void StartVideo( - const media::cast::FrameSenderConfig& config, - const VideoFrameInputAvailableCallback& callback, - const ErrorCallback& error_callback, - const media::cast::CreateVideoEncodeAcceleratorCallback& create_vea_cb, - const media::cast::CreateVideoEncodeMemoryCallback& - create_video_encode_mem_cb); - - // Start remoting session for one stream. After calling this method, a - // remoting sender will be ready for sending the demuxed stream. StartUDP() - // must be called before calling this method. - void StartRemotingStream(int32_t stream_id, - const media::cast::FrameSenderConfig& config, - ErrorOnceCallback error_callback); - - void ToggleLogging(bool is_audio, bool enable); - void GetEventLogsAndReset(bool is_audio, - const std::string& extra_data, const EventLogsCallback& callback); - void GetStatsAndReset(bool is_audio, const StatsCallback& callback); - - protected: - // Called to report back operational status changes. The first time this is - // called with STATUS_INITIALIZED will result in running the "frame input - // available" callback, to indicate the session is ready to accept incoming - // audio/video frames. If this is called with an error that has halted the - // session, the |error_callback| provided to StartXXX() will be run. This - // method may be called multiple times during the session to indicate codec - // re-initializations are taking place and/or runtime errors have occurred. - void OnOperationalStatusChange(bool is_for_audio, - ErrorOnceCallback error_callback, - media::cast::OperationalStatus result); - - private: - void ReceivePacket(std::unique_ptr<media::cast::Packet> packet) override; - - std::unique_ptr<media::cast::CastSender> cast_sender_; - - AudioFrameInputAvailableCallback audio_frame_input_available_callback_; - VideoFrameInputAvailableCallback video_frame_input_available_callback_; - - std::unique_ptr<media::cast::RawEventSubscriberBundle> event_subscribers_; - - base::WeakPtrFactory<CastSessionDelegate> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastSessionDelegate); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_SESSION_DELEGATE_H_ diff --git a/chromium/chrome/renderer/media/cast_threads.cc b/chromium/chrome/renderer/media/cast_threads.cc deleted file mode 100644 index 823df61f05e..00000000000 --- a/chromium/chrome/renderer/media/cast_threads.cc +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_threads.h" - -#include "base/single_thread_task_runner.h" - -CastThreads::CastThreads() - : audio_encode_thread_("CastAudioEncodeThread"), - video_encode_thread_("CastVideoEncodeThread") { - audio_encode_thread_.Start(); - video_encode_thread_.Start(); -} - -scoped_refptr<base::SingleThreadTaskRunner> -CastThreads::GetAudioEncodeTaskRunner() { - return audio_encode_thread_.task_runner(); -} - -scoped_refptr<base::SingleThreadTaskRunner> -CastThreads::GetVideoEncodeTaskRunner() { - return video_encode_thread_.task_runner(); -} diff --git a/chromium/chrome/renderer/media/cast_threads.h b/chromium/chrome/renderer/media/cast_threads.h deleted file mode 100644 index 01f05552a0c..00000000000 --- a/chromium/chrome/renderer/media/cast_threads.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Manages threads used by Cast Streaming Extensions API. There is a -// singleton object of this class in the renderer. -// -// There are two threads owned by this class: -// 1. Audio encode thread. -// 2. Video encode thread. -// These two threads are started this object is created. - -#ifndef CHROME_RENDERER_MEDIA_CAST_THREADS_H_ -#define CHROME_RENDERER_MEDIA_CAST_THREADS_H_ - -#include "base/lazy_instance.h" -#include "base/macros.h" -#include "base/single_thread_task_runner.h" -#include "base/threading/thread.h" - -class CastThreads { - public: - scoped_refptr<base::SingleThreadTaskRunner> GetAudioEncodeTaskRunner(); - scoped_refptr<base::SingleThreadTaskRunner> GetVideoEncodeTaskRunner(); - - private: - friend struct base::LazyInstanceTraitsBase<CastThreads>; - - CastThreads(); - - base::Thread audio_encode_thread_; - base::Thread video_encode_thread_; - - DISALLOW_COPY_AND_ASSIGN(CastThreads); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_THREADS_H_ diff --git a/chromium/chrome/renderer/media/cast_transport_ipc.cc b/chromium/chrome/renderer/media/cast_transport_ipc.cc deleted file mode 100644 index 06842f3bd02..00000000000 --- a/chromium/chrome/renderer/media/cast_transport_ipc.cc +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_transport_ipc.h" - -#include <memory> -#include <utility> - -#include "base/callback.h" -#include "base/containers/id_map.h" -#include "chrome/common/cast_messages.h" -#include "chrome/renderer/media/cast_ipc_dispatcher.h" -#include "ipc/ipc_channel_proxy.h" -#include "media/cast/cast_sender.h" - -CastTransportIPC::CastTransportIPC( - const net::IPEndPoint& local_end_point, - const net::IPEndPoint& remote_end_point, - std::unique_ptr<base::DictionaryValue> options, - media::cast::PacketReceiverCallback packet_callback, - media::cast::CastTransportStatusCallback status_callback, - BulkRawEventsCallback raw_events_cb) - : channel_id_(-1), - packet_callback_(std::move(packet_callback)), - status_callback_(std::move(status_callback)), - raw_events_callback_(std::move(raw_events_cb)) { - if (CastIPCDispatcher::Get()) { - // TODO(miu): CastIPCDispatcher should be provided as a ctor argument. - channel_id_ = CastIPCDispatcher::Get()->AddSender(this); - Send(new CastHostMsg_New(channel_id_, local_end_point, remote_end_point, - *options)); - } -} - -CastTransportIPC::~CastTransportIPC() { - Send(new CastHostMsg_Delete(channel_id_)); - if (CastIPCDispatcher::Get()) { - CastIPCDispatcher::Get()->RemoveSender(channel_id_); - } -} - -void CastTransportIPC::InitializeStream( - const media::cast::CastTransportRtpConfig& config, - std::unique_ptr<media::cast::RtcpObserver> rtcp_observer) { - if (rtcp_observer) { - DCHECK(clients_.find(config.ssrc) == clients_.end()); - clients_[config.ssrc] = std::move(rtcp_observer); - } - Send(new CastHostMsg_InitializeStream(channel_id_, config)); -} - -void CastTransportIPC::InsertFrame(uint32_t ssrc, - const media::cast::EncodedFrame& frame) { - Send(new CastHostMsg_InsertFrame(channel_id_, ssrc, frame)); -} - -void CastTransportIPC::SendSenderReport( - uint32_t ssrc, - base::TimeTicks current_time, - media::cast::RtpTimeTicks current_time_as_rtp_timestamp) { - Send(new CastHostMsg_SendSenderReport(channel_id_, ssrc, current_time, - current_time_as_rtp_timestamp)); -} - -void CastTransportIPC::CancelSendingFrames( - uint32_t ssrc, - const std::vector<media::cast::FrameId>& frame_ids) { - Send(new CastHostMsg_CancelSendingFrames(channel_id_, ssrc, frame_ids)); -} - -void CastTransportIPC::ResendFrameForKickstart(uint32_t ssrc, - media::cast::FrameId frame_id) { - Send(new CastHostMsg_ResendFrameForKickstart(channel_id_, ssrc, frame_id)); -} - -void CastTransportIPC::AddValidRtpReceiver(uint32_t rtp_sender_ssrc, - uint32_t rtp_receiver_ssrc) { - Send(new CastHostMsg_AddValidRtpReceiver(channel_id_, rtp_sender_ssrc, - rtp_receiver_ssrc)); -} - -void CastTransportIPC::InitializeRtpReceiverRtcpBuilder( - uint32_t rtp_receiver_ssrc, - const media::cast::RtcpTimeData& time_data) { - Send(new CastHostMsg_InitializeRtpReceiverRtcpBuilder( - channel_id_, rtp_receiver_ssrc, time_data)); -} - -void CastTransportIPC::AddCastFeedback( - const media::cast::RtcpCastMessage& cast_message, - base::TimeDelta target_delay) { - Send( - new CastHostMsg_AddCastFeedback(channel_id_, cast_message, target_delay)); -} - -void CastTransportIPC::AddPli(const media::cast::RtcpPliMessage& pli_message) { - Send(new CastHostMsg_AddPli(channel_id_, pli_message)); -} - -void CastTransportIPC::AddRtcpEvents( - const media::cast::ReceiverRtcpEventSubscriber::RtcpEvents& rtcp_events) { - Send(new CastHostMsg_AddRtcpEvents(channel_id_, rtcp_events)); -} - -void CastTransportIPC::AddRtpReceiverReport( - const media::cast::RtcpReportBlock& rtp_receiver_report_block) { - Send(new CastHostMsg_AddRtpReceiverReport(channel_id_, - rtp_receiver_report_block)); -} - -void CastTransportIPC::SendRtcpFromRtpReceiver() { - Send(new CastHostMsg_SendRtcpFromRtpReceiver(channel_id_)); -} - -void CastTransportIPC::OnNotifyStatusChange( - media::cast::CastTransportStatus status) { - status_callback_.Run(status); -} - -void CastTransportIPC::OnRawEvents( - const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events) { - // Note: Casting away const to avoid having to copy all the data elements. As - // the only consumer of this data in the IPC message, mutating the inputs - // should be acceptable. Just nod and blame the interface we were given here. - std::unique_ptr<std::vector<media::cast::FrameEvent>> taken_frame_events( - new std::vector<media::cast::FrameEvent>()); - taken_frame_events->swap( - const_cast<std::vector<media::cast::FrameEvent>&>(frame_events)); - std::unique_ptr<std::vector<media::cast::PacketEvent>> taken_packet_events( - new std::vector<media::cast::PacketEvent>()); - taken_packet_events->swap( - const_cast<std::vector<media::cast::PacketEvent>&>(packet_events)); - raw_events_callback_.Run(std::move(taken_frame_events), - std::move(taken_packet_events)); -} - -void CastTransportIPC::OnRtt(uint32_t rtp_sender_ssrc, base::TimeDelta rtt) { - auto it = clients_.find(rtp_sender_ssrc); - if (it == clients_.end()) { - LOG(ERROR) << "Received RTT report for unknown SSRC: " << rtp_sender_ssrc; - return; - } - it->second->OnReceivedRtt(rtt); -} - -void CastTransportIPC::OnRtcpCastMessage( - uint32_t rtp_sender_ssrc, - const media::cast::RtcpCastMessage& cast_message) { - auto it = clients_.find(rtp_sender_ssrc); - if (it == clients_.end()) { - LOG(ERROR) << "Received cast message for unknown SSRC: " << rtp_sender_ssrc; - return; - } - it->second->OnReceivedCastMessage(cast_message); -} - -void CastTransportIPC::OnReceivedPli(uint32_t rtp_sender_ssrc) { - auto it = clients_.find(rtp_sender_ssrc); - if (it == clients_.end()) { - LOG(ERROR) << "Received picture loss indicator for unknown SSRC: " - << rtp_sender_ssrc; - return; - } - it->second->OnReceivedPli(); -} - -void CastTransportIPC::OnReceivedPacket(const media::cast::Packet& packet) { - if (!packet_callback_.is_null()) { - // TODO(hubbe): Perhaps an non-ownership-transferring cb here? - std::unique_ptr<media::cast::Packet> packet_copy( - new media::cast::Packet(packet)); - packet_callback_.Run(std::move(packet_copy)); - } else { - DVLOG(1) << "CastIPCDispatcher::OnReceivedPacket no packet callback yet."; - } -} - -void CastTransportIPC::Send(IPC::Message* message) { - if (CastIPCDispatcher::Get() && channel_id_ != -1) { - CastIPCDispatcher::Get()->Send(message); - } else { - delete message; - } -} diff --git a/chromium/chrome/renderer/media/cast_transport_ipc.h b/chromium/chrome/renderer/media/cast_transport_ipc.h deleted file mode 100644 index 3ee563bd636..00000000000 --- a/chromium/chrome/renderer/media/cast_transport_ipc.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2014 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_TRANSPORT_IPC_H_ -#define CHROME_RENDERER_MEDIA_CAST_TRANSPORT_IPC_H_ - -#include <stdint.h> - -#include <map> -#include <memory> - -#include "base/macros.h" -#include "base/threading/thread_task_runner_handle.h" -#include "ipc/ipc_channel_proxy.h" -#include "media/cast/logging/logging_defines.h" -#include "media/cast/net/cast_transport.h" - -// This implementation of the CastTransport interface communicates with the -// browser process over IPC and relays all calls to/from the cast transport to -// the browser process. The primary reason for this arrangement is to give the -// renderer less direct control over the UDP sockets. -class CastTransportIPC : public media::cast::CastTransport { - public: - // Runs when a raw event completes. - using BulkRawEventsCallback = base::RepeatingCallback<void( - std::unique_ptr<std::vector<media::cast::FrameEvent>>, - std::unique_ptr<std::vector<media::cast::PacketEvent>>)>; - - CastTransportIPC(const net::IPEndPoint& local_end_point, - const net::IPEndPoint& remote_end_point, - std::unique_ptr<base::DictionaryValue> options, - media::cast::PacketReceiverCallback packet_callback, - media::cast::CastTransportStatusCallback status_callback, - BulkRawEventsCallback raw_events_cb); - - ~CastTransportIPC() override; - - // media::cast::CastTransport implementation. - void InitializeStream( - const media::cast::CastTransportRtpConfig& config, - std::unique_ptr<media::cast::RtcpObserver> rtcp_observer) override; - void InsertFrame(uint32_t ssrc, - const media::cast::EncodedFrame& frame) override; - void SendSenderReport( - uint32_t ssrc, - base::TimeTicks current_time, - media::cast::RtpTimeTicks current_time_as_rtp_timestamp) override; - void CancelSendingFrames( - uint32_t ssrc, - const std::vector<media::cast::FrameId>& frame_ids) override; - void ResendFrameForKickstart(uint32_t ssrc, - media::cast::FrameId frame_id) override; - void AddValidRtpReceiver(uint32_t rtp_sender_ssrc, - uint32_t rtp_receiver_ssrc) override; - void InitializeRtpReceiverRtcpBuilder( - uint32_t rtp_receiver_ssrc, - const media::cast::RtcpTimeData& time_data) override; - void AddCastFeedback(const media::cast::RtcpCastMessage& cast_message, - base::TimeDelta target_delay) override; - void AddPli(const media::cast::RtcpPliMessage& pli_message) override; - void AddRtcpEvents(const media::cast::ReceiverRtcpEventSubscriber::RtcpEvents& - rtcp_events) override; - void AddRtpReceiverReport( - const media::cast::RtcpReportBlock& rtp_receiver_report_block) override; - void SendRtcpFromRtpReceiver() override; - void SetOptions(const base::DictionaryValue& options) final {} - void OnNotifyStatusChange(media::cast::CastTransportStatus status); - void OnRawEvents(const std::vector<media::cast::PacketEvent>& packet_events, - const std::vector<media::cast::FrameEvent>& frame_events); - void OnRtt(uint32_t rtp_sender_ssrc, base::TimeDelta rtt); - void OnRtcpCastMessage(uint32_t rtp_sender_ssrc, - const media::cast::RtcpCastMessage& cast_message); - void OnReceivedPli(uint32_t rtp_sender_ssrc); - void OnReceivedPacket(const media::cast::Packet& packet); - - private: - void Send(IPC::Message* message); - - int32_t channel_id_; - media::cast::PacketReceiverCallback packet_callback_; - media::cast::CastTransportStatusCallback status_callback_; - const BulkRawEventsCallback raw_events_callback_; - using ClientMap = - std::map<uint32_t, std::unique_ptr<media::cast::RtcpObserver>>; - ClientMap clients_; - - DISALLOW_COPY_AND_ASSIGN(CastTransportIPC); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_TRANSPORT_IPC_H_ diff --git a/chromium/chrome/renderer/media/cast_udp_transport.cc b/chromium/chrome/renderer/media/cast_udp_transport.cc deleted file mode 100644 index a8302a8a0fa..00000000000 --- a/chromium/chrome/renderer/media/cast_udp_transport.cc +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/media/cast_udp_transport.h" - -#include <memory> -#include <utility> - -#include "base/memory/ptr_util.h" -#include "base/values.h" -#include "chrome/renderer/media/cast_session.h" - -CastUdpTransport::CastUdpTransport(const scoped_refptr<CastSession>& session) - : cast_session_(session), options_(new base::DictionaryValue) {} - -CastUdpTransport::~CastUdpTransport() { -} - -void CastUdpTransport::SetDestination( - const net::IPEndPoint& remote_address, - const CastSessionDelegate::ErrorCallback& error_callback) { - DVLOG(1) << "CastUdpTransport::SetDestination = " - << remote_address.ToString(); - remote_address_ = remote_address; - cast_session_->StartUDP( - remote_address, base::WrapUnique(options_->DeepCopy()), error_callback); -} - -void CastUdpTransport::SetOptions( - std::unique_ptr<base::DictionaryValue> options) { - options_ = std::move(options); -} diff --git a/chromium/chrome/renderer/media/cast_udp_transport.h b/chromium/chrome/renderer/media/cast_udp_transport.h deleted file mode 100644 index 726edbe5c67..00000000000 --- a/chromium/chrome/renderer/media/cast_udp_transport.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CHROME_RENDERER_MEDIA_CAST_UDP_TRANSPORT_H_ -#define CHROME_RENDERER_MEDIA_CAST_UDP_TRANSPORT_H_ - -#include <memory> - -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "chrome/renderer/media/cast_session_delegate.h" -#include "net/base/ip_endpoint.h" - -namespace base { -class DictionaryValue; -} // namespace base - -class CastSession; - -// This class represents the transport mechanism used by Cast RTP streams -// to connect to a remote client. It specifies the destination address -// and network protocol used to send Cast RTP streams. -class CastUdpTransport { - public: - explicit CastUdpTransport(const scoped_refptr<CastSession>& session); - virtual ~CastUdpTransport(); - - // Specify the remote IP address and port. - void SetDestination(const net::IPEndPoint& remote_address, - const CastSessionDelegate::ErrorCallback& error_callback); - - // Set options. - void SetOptions(std::unique_ptr<base::DictionaryValue> options); - - private: - const scoped_refptr<CastSession> cast_session_; - net::IPEndPoint remote_address_; - std::unique_ptr<base::DictionaryValue> options_; - base::WeakPtrFactory<CastUdpTransport> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(CastUdpTransport); -}; - -#endif // CHROME_RENDERER_MEDIA_CAST_UDP_TRANSPORT_H_ diff --git a/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc b/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc index 2b35d6d399a..70da74b25e7 100644 --- a/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc +++ b/chromium/chrome/renderer/media/chrome_speech_recognition_client.cc @@ -6,22 +6,60 @@ #include <utility> +#include "base/metrics/field_trial_params.h" #include "content/public/renderer/render_frame.h" +#include "media/base/audio_bus.h" +#include "media/base/audio_parameters.h" +#include "media/base/bind_to_current_loop.h" +#include "media/base/channel_mixer.h" +#include "media/base/media_switches.h" #include "media/mojo/mojom/media_types.mojom.h" #include "third_party/blink/public/common/browser_interface_broker_proxy.h" +#include "third_party/blink/public/platform/web_string.h" +#include "third_party/blink/public/web/web_frame.h" +#include "third_party/blink/public/web/web_local_frame.h" + +// Get the list of blocked URLs defined by the Finch experiment parameter. These +// websites provide captions by default and thus do not require the live caption +// feature. +std::vector<std::string> GetBlockedURLs() { + return base::SplitString(base::GetFieldTrialParamValueByFeature( + media::kLiveCaption, "blocked_websites"), + ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL); +} ChromeSpeechRecognitionClient::ChromeSpeechRecognitionClient( - content::RenderFrame* render_frame) { + content::RenderFrame* render_frame, + media::SpeechRecognitionClient::OnReadyCallback callback) + : on_ready_callback_(std::move(callback)), blocked_urls_(GetBlockedURLs()) { mojo::PendingReceiver<media::mojom::SpeechRecognitionContext> speech_recognition_context_receiver = speech_recognition_context_.BindNewPipeAndPassReceiver(); speech_recognition_context_->BindRecognizer( speech_recognition_recognizer_.BindNewPipeAndPassReceiver(), - speech_recognition_client_receiver_.BindNewPipeAndPassRemote()); + speech_recognition_client_receiver_.BindNewPipeAndPassRemote(), + base::BindOnce(&ChromeSpeechRecognitionClient::OnRecognizerBound, + base::Unretained(this))); + render_frame->GetBrowserInterfaceBroker()->GetInterface( std::move(speech_recognition_context_receiver)); render_frame->GetBrowserInterfaceBroker()->GetInterface( caption_host_.BindNewPipeAndPassReceiver()); + is_website_blocked_ = IsUrlBlocked( + render_frame->GetWebFrame()->GetSecurityOrigin().ToString().Utf8()); + + send_audio_callback_ = media::BindToCurrentLoop(base::BindRepeating( + &ChromeSpeechRecognitionClient::SendAudioToSpeechRecognitionService, + weak_factory_.GetWeakPtr())); +} + +void ChromeSpeechRecognitionClient::OnRecognizerBound( + bool is_multichannel_supported) { + is_multichannel_supported_ = is_multichannel_supported; + is_recognizer_bound_ = true; + + if (on_ready_callback_) + std::move(on_ready_callback_).Run(); } ChromeSpeechRecognitionClient::~ChromeSpeechRecognitionClient() = default; @@ -29,21 +67,85 @@ ChromeSpeechRecognitionClient::~ChromeSpeechRecognitionClient() = default; void ChromeSpeechRecognitionClient::AddAudio( scoped_refptr<media::AudioBuffer> buffer) { DCHECK(buffer); - if (IsSpeechRecognitionAvailable()) { - speech_recognition_recognizer_->SendAudioToSpeechRecognitionService( - ConvertToAudioDataS16(std::move(buffer))); - } + send_audio_callback_.Run(ConvertToAudioDataS16(std::move(buffer))); +} + +void ChromeSpeechRecognitionClient::AddAudio( + std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout) { + DCHECK(audio_bus); + send_audio_callback_.Run( + ConvertToAudioDataS16(std::move(audio_bus), sample_rate, channel_layout)); } bool ChromeSpeechRecognitionClient::IsSpeechRecognitionAvailable() { - return speech_recognition_recognizer_.is_bound() && - speech_recognition_recognizer_.is_connected(); + // TODO(evliu): Check if SODA is available. + return !is_website_blocked_ && is_browser_requesting_transcription_ && + is_recognizer_bound_; +} + +// The OnReadyCallback is set by the owner of |this| and is executed when speech +// recognition becomes available. Setting the callback will override any +// existing callback. +void ChromeSpeechRecognitionClient::SetOnReadyCallback( + SpeechRecognitionClient::OnReadyCallback callback) { + on_ready_callback_ = std::move(callback); + + // Immediately run the callback if speech recognition is already available. + if (IsSpeechRecognitionAvailable() && on_ready_callback_) + std::move(on_ready_callback_).Run(); } void ChromeSpeechRecognitionClient::OnSpeechRecognitionRecognitionEvent( media::mojom::SpeechRecognitionResultPtr result) { - caption_host_->OnTranscription(chrome::mojom::TranscriptionResult::New( - result->transcription, result->is_final)); + caption_host_->OnTranscription( + chrome::mojom::TranscriptionResult::New(result->transcription, + result->is_final), + base::BindOnce(&ChromeSpeechRecognitionClient::OnTranscriptionCallback, + base::Unretained(this))); +} + +void ChromeSpeechRecognitionClient::OnTranscriptionCallback(bool success) { + is_browser_requesting_transcription_ = success; +} + +void ChromeSpeechRecognitionClient::CopyBufferToTempAudioBus( + const media::AudioBuffer& buffer) { + if (!temp_audio_bus_ || + buffer.channel_count() != temp_audio_bus_->channels() || + buffer.frame_count() != temp_audio_bus_->frames()) { + temp_audio_bus_ = + media::AudioBus::Create(buffer.channel_count(), buffer.frame_count()); + } + + buffer.ReadFrames(buffer.frame_count(), + /* source_frame_offset */ 0, /* dest_frame_offset */ 0, + temp_audio_bus_.get()); +} + +void ChromeSpeechRecognitionClient::ResetChannelMixer( + int frame_count, + media::ChannelLayout channel_layout) { + if (!monaural_audio_bus_ || frame_count != monaural_audio_bus_->frames()) { + monaural_audio_bus_ = + media::AudioBus::Create(1 /* channels */, frame_count); + } + + if (channel_layout != channel_layout_) { + channel_layout_ = channel_layout; + channel_mixer_ = std::make_unique<media::ChannelMixer>( + channel_layout, media::CHANNEL_LAYOUT_MONO); + } +} + +void ChromeSpeechRecognitionClient::SendAudioToSpeechRecognitionService( + media::mojom::AudioDataS16Ptr audio_data) { + DCHECK(audio_data); + if (IsSpeechRecognitionAvailable()) { + speech_recognition_recognizer_->SendAudioToSpeechRecognitionService( + std::move(audio_data)); + } } media::mojom::AudioDataS16Ptr @@ -58,6 +160,21 @@ ChromeSpeechRecognitionClient::ConvertToAudioDataS16( signed_buffer->frame_count = buffer->frame_count(); signed_buffer->sample_rate = buffer->sample_rate(); + // If multichannel audio is not supported by the speech recognition service, + // mix the channels into a monaural channel before converting it. + if (buffer->channel_count() > 1 && !is_multichannel_supported_) { + signed_buffer->channel_count = 1; + CopyBufferToTempAudioBus(*buffer); + ResetChannelMixer(buffer->frame_count(), buffer->channel_layout()); + signed_buffer->data.resize(buffer->frame_count()); + channel_mixer_->Transform(temp_audio_bus_.get(), monaural_audio_bus_.get()); + monaural_audio_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>( + monaural_audio_bus_->frames(), &signed_buffer->data[0]); + return signed_buffer; + } + + // If the audio is already in the interleaved signed int 16 format, directly + // assign it to the buffer. if (buffer->sample_format() == media::SampleFormat::kSampleFormatS16) { int16_t* audio_data = reinterpret_cast<int16_t*>(buffer->channel_data()[0]); signed_buffer->data.assign( @@ -67,20 +184,48 @@ ChromeSpeechRecognitionClient::ConvertToAudioDataS16( } // Convert the raw audio to the interleaved signed int 16 sample type. - if (!temp_audio_bus_ || - buffer->channel_count() != temp_audio_bus_->channels() || - buffer->frame_count() != temp_audio_bus_->frames()) { - temp_audio_bus_ = - media::AudioBus::Create(buffer->channel_count(), buffer->frame_count()); - } - - buffer->ReadFrames(buffer->frame_count(), - /* source_frame_offset */ 0, /* dest_frame_offset */ 0, - temp_audio_bus_.get()); - + CopyBufferToTempAudioBus(*buffer); signed_buffer->data.resize(buffer->frame_count() * buffer->channel_count()); temp_audio_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>( temp_audio_bus_->frames(), &signed_buffer->data[0]); return signed_buffer; } + +media::mojom::AudioDataS16Ptr +ChromeSpeechRecognitionClient::ConvertToAudioDataS16( + std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout) { + DCHECK_GT(audio_bus->frames(), 0); + DCHECK_GT(audio_bus->channels(), 0); + + auto signed_buffer = media::mojom::AudioDataS16::New(); + signed_buffer->channel_count = audio_bus->channels(); + signed_buffer->frame_count = audio_bus->frames(); + signed_buffer->sample_rate = sample_rate; + + // If multichannel audio is not supported by the speech recognition service, + // mix the channels into a monaural channel before converting it. + if (audio_bus->channels() > 1 && !is_multichannel_supported_) { + signed_buffer->channel_count = 1; + ResetChannelMixer(audio_bus->frames(), channel_layout); + signed_buffer->data.resize(audio_bus->frames()); + + channel_mixer_->Transform(audio_bus.get(), monaural_audio_bus_.get()); + monaural_audio_bus_->ToInterleaved<media::SignedInt16SampleTypeTraits>( + monaural_audio_bus_->frames(), &signed_buffer->data[0]); + + return signed_buffer; + } + + signed_buffer->data.resize(audio_bus->frames() * audio_bus->channels()); + audio_bus->ToInterleaved<media::SignedInt16SampleTypeTraits>( + audio_bus->frames(), &signed_buffer->data[0]); + + return signed_buffer; +} + +bool ChromeSpeechRecognitionClient::IsUrlBlocked(const std::string& url) const { + return blocked_urls_.find(url) != blocked_urls_.end(); +} diff --git a/chromium/chrome/renderer/media/chrome_speech_recognition_client.h b/chromium/chrome/renderer/media/chrome_speech_recognition_client.h index 5f08b42cfd9..df0e0574a87 100644 --- a/chromium/chrome/renderer/media/chrome_speech_recognition_client.h +++ b/chromium/chrome/renderer/media/chrome_speech_recognition_client.h @@ -8,6 +8,8 @@ #include <memory> #include <string> +#include "base/containers/flat_set.h" +#include "base/memory/weak_ptr.h" #include "chrome/common/caption.mojom.h" #include "media/base/audio_buffer.h" #include "media/base/speech_recognition_client.h" @@ -19,11 +21,21 @@ namespace content { class RenderFrame; } // namespace content +namespace media { +class AudioBus; +class ChannelMixer; +} // namespace media + class ChromeSpeechRecognitionClient : public media::SpeechRecognitionClient, public media::mojom::SpeechRecognitionRecognizerClient { public: - explicit ChromeSpeechRecognitionClient(content::RenderFrame* render_frame); + using SendAudioToSpeechRecognitionServiceCallback = + base::RepeatingCallback<void(media::mojom::AudioDataS16Ptr audio_data)>; + + explicit ChromeSpeechRecognitionClient( + content::RenderFrame* render_frame, + media::SpeechRecognitionClient::OnReadyCallback callback); ChromeSpeechRecognitionClient(const ChromeSpeechRecognitionClient&) = delete; ChromeSpeechRecognitionClient& operator=( const ChromeSpeechRecognitionClient&) = delete; @@ -31,16 +43,51 @@ class ChromeSpeechRecognitionClient // media::SpeechRecognitionClient void AddAudio(scoped_refptr<media::AudioBuffer> buffer) override; + void AddAudio(std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout) override; bool IsSpeechRecognitionAvailable() override; + void SetOnReadyCallback( + SpeechRecognitionClient::OnReadyCallback callback) override; + + // Callback executed when the recognizer is bound. Sets the flag indicating + // whether the speech recognition service supports multichannel audio. + void OnRecognizerBound(bool is_multichannel_supported); // media::mojom::SpeechRecognitionRecognizerClient void OnSpeechRecognitionRecognitionEvent( media::mojom::SpeechRecognitionResultPtr result) override; private: + void SendAudioToSpeechRecognitionService( + media::mojom::AudioDataS16Ptr audio_data); + media::mojom::AudioDataS16Ptr ConvertToAudioDataS16( scoped_refptr<media::AudioBuffer> buffer); + // Called as a response to sending a transcription to the browser. + void OnTranscriptionCallback(bool success); + + media::mojom::AudioDataS16Ptr ConvertToAudioDataS16( + std::unique_ptr<media::AudioBus> audio_bus, + int sample_rate, + media::ChannelLayout channel_layout); + + // Recreates the temporary audio bus if the frame count or channel count + // changed and reads the frames from the buffer into the temporary audio bus. + void CopyBufferToTempAudioBus(const media::AudioBuffer& buffer); + + // Resets the temporary monaural audio bus and the channel mixer used to + // combine multiple audio channels. + void ResetChannelMixer(int frame_count, media::ChannelLayout channel_layout); + + bool IsUrlBlocked(const std::string& url) const; + + media::SpeechRecognitionClient::OnReadyCallback on_ready_callback_; + + // Sends audio to the speech recognition thread on the renderer thread. + SendAudioToSpeechRecognitionServiceCallback send_audio_callback_; + mojo::Remote<media::mojom::SpeechRecognitionContext> speech_recognition_context_; mojo::Remote<media::mojom::SpeechRecognitionRecognizer> @@ -49,9 +96,33 @@ class ChromeSpeechRecognitionClient speech_recognition_client_receiver_{this}; mojo::Remote<chrome::mojom::CaptionHost> caption_host_; + bool is_website_blocked_ = false; + const base::flat_set<std::string> blocked_urls_; + // The temporary audio bus used to convert the raw audio to the appropriate // format. std::unique_ptr<media::AudioBus> temp_audio_bus_; + + // Whether the browser is still requesting transcriptions. + bool is_browser_requesting_transcription_ = true; + + bool is_recognizer_bound_ = false; + + // The temporary audio bus used to mix multichannel audio into a single + // channel. + std::unique_ptr<media::AudioBus> monaural_audio_bus_; + + std::unique_ptr<media::ChannelMixer> channel_mixer_; + + // The layout used to instantiate the channel mixer. + media::ChannelLayout channel_layout_ = + media::ChannelLayout::CHANNEL_LAYOUT_NONE; + + // A flag indicating whether the speech recognition service supports + // multichannel audio. + bool is_multichannel_supported_ = false; + + base::WeakPtrFactory<ChromeSpeechRecognitionClient> weak_factory_{this}; }; #endif // CHROME_RENDERER_MEDIA_CHROME_SPEECH_RECOGNITION_CLIENT_H_ diff --git a/chromium/chrome/renderer/media/media_feeds.cc b/chromium/chrome/renderer/media/media_feeds.cc index cdbc1995457..fae6fdeb893 100644 --- a/chromium/chrome/renderer/media/media_feeds.cc +++ b/chromium/chrome/renderer/media/media_feeds.cc @@ -45,14 +45,9 @@ base::Optional<GURL> MediaFeeds::GetMediaFeedURL(content::RenderFrame* frame) { if (!elem.HasHTMLTagName("link")) continue; - // The <link> rel must be feed. + // The <link> rel must be media-feed. std::string rel = elem.GetAttribute("rel").Utf8(); - if (!base::LowerCaseEqualsASCII(rel, "feed")) - continue; - - // The <link> type must the JSON+LD mime type. - std::string type = elem.GetAttribute("type").Utf8(); - if (!base::LowerCaseEqualsASCII(type, "application/ld+json")) + if (!base::LowerCaseEqualsASCII(rel, "media-feed")) continue; WebString href = elem.GetAttribute("href"); diff --git a/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc b/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc index b8b24d4ebb9..d2495adc221 100644 --- a/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc +++ b/chromium/chrome/renderer/media/webrtc_logging_agent_impl.cc @@ -82,6 +82,7 @@ void WebRtcLoggingAgentImpl::AddReceiver( void WebRtcLoggingAgentImpl::Start( mojo::PendingRemote<mojom::WebRtcLoggingClient> pending_client) { // We only support one client at a time. OK to drop any existing client. + client_.reset(); client_.Bind(std::move(pending_client)); WebRtcLogMessageDelegateImpl::GetInstance()->Start(base::BindRepeating( diff --git a/chromium/chrome/renderer/net/net_error_helper.cc b/chromium/chrome/renderer/net/net_error_helper.cc index b09c9aacc91..f9c56beebe6 100644 --- a/chromium/chrome/renderer/net/net_error_helper.cc +++ b/chromium/chrome/renderer/net/net_error_helper.cc @@ -225,13 +225,7 @@ void NetErrorHelper::DidStartNavigation( core_->OnStartLoad(GetFrameType(render_frame()), GetLoadingPageType(url)); } -void NetErrorHelper::DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) { - // If this is a "same-document" navigation, it's not a real navigation. There - // wasn't a start event for it, either, so just ignore it. - if (is_same_document_navigation) - return; - +void NetErrorHelper::DidCommitProvisionalLoad(ui::PageTransition transition) { // Invalidate weak pointers from old error page controllers. If loading a new // error page, the controller has not yet been attached, so this won't affect // it. @@ -323,6 +317,15 @@ chrome::mojom::NetworkEasterEgg* NetErrorHelper::GetRemoteNetworkEasterEgg() { return remote_network_easter_egg_.get(); } +chrome::mojom::NetErrorPageSupport* +NetErrorHelper::GetRemoteNetErrorPageSupport() { + if (!remote_net_error_page_support_) { + render_frame()->GetRemoteAssociatedInterfaces()->GetInterface( + &remote_net_error_page_support_); + } + return remote_net_error_page_support_.get(); +} + LocalizedError::PageState NetErrorHelper::GenerateLocalizedErrorPage( const error_page::Error& error, bool is_failed_post, @@ -476,18 +479,15 @@ void NetErrorHelper::DiagnoseError(const GURL& page_url) { } void NetErrorHelper::DownloadPageLater() { -#if defined(OS_ANDROID) - render_frame()->Send(new ChromeViewHostMsg_DownloadPageLater( - render_frame()->GetRoutingID())); -#endif // defined(OS_ANDROID) +#if BUILDFLAG(ENABLE_OFFLINE_PAGES) + GetRemoteNetErrorPageSupport()->DownloadPageLater(); +#endif // BUILDFLAG(ENABLE_OFFLINE_PAGES) } void NetErrorHelper::SetIsShowingDownloadButton(bool show) { -#if defined(OS_ANDROID) - render_frame()->Send( - new ChromeViewHostMsg_SetIsShowingDownloadButtonInErrorPage( - render_frame()->GetRoutingID(), show)); -#endif // defined(OS_ANDROID) +#if BUILDFLAG(ENABLE_OFFLINE_PAGES) + GetRemoteNetErrorPageSupport()->SetIsShowingDownloadButtonInErrorPage(show); +#endif // BUILDFLAG(ENABLE_OFFLINE_PAGES) } void NetErrorHelper::OfflineContentAvailable( diff --git a/chromium/chrome/renderer/net/net_error_helper.h b/chromium/chrome/renderer/net/net_error_helper.h index 00ae60941fe..6ce1ed95f09 100644 --- a/chromium/chrome/renderer/net/net_error_helper.h +++ b/chromium/chrome/renderer/net/net_error_helper.h @@ -13,6 +13,7 @@ #include "base/memory/weak_ptr.h" #include "build/build_config.h" #include "chrome/common/navigation_corrector.mojom.h" +#include "chrome/common/net/net_error_page_support.mojom.h" #include "chrome/common/network_diagnostics.mojom.h" #include "chrome/common/network_easter_egg.mojom.h" #include "chrome/renderer/net/net_error_helper_core.h" @@ -27,6 +28,7 @@ #include "mojo/public/cpp/bindings/associated_receiver_set.h" #include "mojo/public/cpp/bindings/associated_remote.h" #include "mojo/public/cpp/bindings/pending_associated_receiver.h" +#include "mojo/public/cpp/bindings/remote.h" #include "net/base/net_errors.h" class GURL; @@ -80,8 +82,7 @@ class NetErrorHelper void DidStartNavigation( const GURL& url, base::Optional<blink::WebNavigationType> navigation_type) override; - void DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) override; + void DidCommitProvisionalLoad(ui::PageTransition transition) override; void DidFinishLoad() override; void OnStop() override; void WasShown() override; @@ -110,6 +111,7 @@ class NetErrorHelper const GURL& url) const; chrome::mojom::NetworkDiagnostics* GetRemoteNetworkDiagnostics(); chrome::mojom::NetworkEasterEgg* GetRemoteNetworkEasterEgg(); + chrome::mojom::NetErrorPageSupport* GetRemoteNetErrorPageSupport(); // NetErrorHelperCore::Delegate implementation: error_page::LocalizedError::PageState GenerateLocalizedErrorPage( @@ -189,6 +191,8 @@ class NetErrorHelper navigation_corrector_receivers_; mojo::AssociatedRemote<chrome::mojom::NetworkEasterEgg> remote_network_easter_egg_; + mojo::AssociatedRemote<chrome::mojom::NetErrorPageSupport> + remote_net_error_page_support_; // Weak factories for vending weak pointers to PageControllers. Weak // pointers are invalidated on each commit, to prevent getting messages from diff --git a/chromium/chrome/renderer/net/net_error_helper_core.cc b/chromium/chrome/renderer/net/net_error_helper_core.cc index 7ce51f9d9d2..e2e56100e63 100644 --- a/chromium/chrome/renderer/net/net_error_helper_core.cc +++ b/chromium/chrome/renderer/net/net_error_helper_core.cc @@ -469,9 +469,8 @@ bool NetErrorHelperCore::IsReloadableError( info.error.reason() != net::ERR_SSL_PROTOCOL_ERROR && // Do not trigger for blacklisted URLs. // https://crbug.com/803839 - info.error.reason() != net::ERR_BLOCKED_BY_ADMINISTRATOR && // Do not trigger for requests that were blocked by the browser itself. - info.error.reason() != net::ERR_BLOCKED_BY_CLIENT && + !net::IsRequestBlockedError(info.error.reason()) && !info.was_failed_post && // Do not trigger for this error code because it is used by Chrome // while an auth prompt is being displayed. @@ -501,7 +500,8 @@ NetErrorHelperCore::NetErrorHelperCore(Delegate* delegate, online_(content::RenderThread::Get()->IsOnline()), visible_(is_visible), auto_reload_count_(0), - navigation_from_button_(NO_BUTTON) + navigation_from_button_(NO_BUTTON), + custom_error_page_(false) #if defined(OS_ANDROID) , page_auto_fetcher_helper_( @@ -662,7 +662,8 @@ void NetErrorHelperCore::OnFinishLoad(FrameType frame_type) { committed_error_page_info_->error, *committed_error_page_info_->navigation_correction_params)); } else if (auto_reload_enabled_ && - IsReloadableError(*committed_error_page_info_)) { + IsReloadableError(*committed_error_page_info_) && + !custom_error_page_) { MaybeStartAutoReloadTimer(); } @@ -691,10 +692,13 @@ void NetErrorHelperCore::PrepareErrorPage(FrameType frame_type, PrepareErrorPageForMainFrame(pending_error_page_info_.get(), error_html); } else { if (error_html) { + custom_error_page_ = false; delegate_->GenerateLocalizedErrorPage( error, is_failed_post, false /* No diagnostics dialogs allowed for subframes. */, nullptr, error_html); + } else { + custom_error_page_ = true; } } } @@ -764,9 +768,12 @@ void NetErrorHelperCore::PrepareErrorPageForMainFrame( error = GetUpdatedError(*pending_error_page_info); } if (error_html) { + custom_error_page_ = false; pending_error_page_info->page_state = delegate_->GenerateLocalizedErrorPage( error, pending_error_page_info->was_failed_post, can_show_network_diagnostics_dialog_, nullptr, error_html); + } else { + custom_error_page_ = true; } } diff --git a/chromium/chrome/renderer/net/net_error_helper_core.h b/chromium/chrome/renderer/net/net_error_helper_core.h index a58cc7ee50b..25862bebe44 100644 --- a/chromium/chrome/renderer/net/net_error_helper_core.h +++ b/chromium/chrome/renderer/net/net_error_helper_core.h @@ -339,6 +339,10 @@ class NetErrorHelperCore { // in errors. Button navigation_from_button_; + // True if the current error page is displaying custom HTML (e.g. + // security interstitials). + bool custom_error_page_; + #if defined(OS_ANDROID) AvailableOfflineContentHelper available_content_helper_; std::unique_ptr<PageAutoFetcherHelper> page_auto_fetcher_helper_; diff --git a/chromium/chrome/renderer/net/net_error_helper_core_unittest.cc b/chromium/chrome/renderer/net/net_error_helper_core_unittest.cc index b7d13ca7f39..1220513a2d3 100644 --- a/chromium/chrome/renderer/net/net_error_helper_core_unittest.cc +++ b/chromium/chrome/renderer/net/net_error_helper_core_unittest.cc @@ -171,7 +171,7 @@ class NetErrorHelperCoreTest : public testing::Test, public NetErrorHelperCore::Delegate { public: NetErrorHelperCoreTest() - : timer_(NULL), + : timer_(nullptr), update_count_(0), error_html_update_count_(0), reload_count_(0), @@ -310,6 +310,20 @@ class NetErrorHelperCoreTest : public testing::Test, DoErrorLoadOfURL(error, GURL(kFailedUrl)); } + void DoErrorLoadWithCustomErrorPage(net::Error error) { + core()->OnStartLoad(NetErrorHelperCore::MAIN_FRAME, + NetErrorHelperCore::NON_ERROR_PAGE); + // Custom error pages pass nullptr to PrepareErrorPage, since they set the + // error HTML on their own. + core()->PrepareErrorPage(NetErrorHelperCore::MAIN_FRAME, + NetErrorForURL(error, GURL(kFailedUrl)), + false /* is_failed_post */, nullptr); + core()->OnStartLoad(NetErrorHelperCore::MAIN_FRAME, + NetErrorHelperCore::ERROR_PAGE); + core()->OnCommitLoad(NetErrorHelperCore::MAIN_FRAME, error_url()); + core()->OnFinishLoad(NetErrorHelperCore::MAIN_FRAME); + } + void DoSuccessLoad() { core()->OnStartLoad(NetErrorHelperCore::MAIN_FRAME, NetErrorHelperCore::NON_ERROR_PAGE); @@ -2421,6 +2435,25 @@ TEST_F(NetErrorHelperCoreAutoReloadTest, ManualReloadShowsError) { net::ERR_CONNECTION_RESET)); } +TEST_F(NetErrorHelperCoreAutoReloadTest, + AutoReloadDisabledForCustomErrorPages) { + // This error code would normally trigger auto-reloads, but shouldn't if we + // are showing a custom error page (e.g. an interstitial.). + DoErrorLoadWithCustomErrorPage(net::ERR_CONNECTION_RESET); + + EXPECT_FALSE(timer()->IsRunning()); + EXPECT_EQ(0, reload_count()); +} + +TEST_F(NetErrorHelperCoreAutoReloadTest, + AutoReloadEnabledAfterCustomErrorPage) { + DoErrorLoadWithCustomErrorPage(net::ERR_CONNECTION_RESET); + EXPECT_FALSE(timer()->IsRunning()); + EXPECT_EQ(0, reload_count()); + DoErrorLoad(net::ERR_CONNECTION_RESET); + EXPECT_TRUE(timer()->IsRunning()); +} + TEST_F(NetErrorHelperCoreTest, ExplicitReloadSucceeds) { DoErrorLoad(net::ERR_CONNECTION_RESET); EXPECT_EQ(0, reload_count()); diff --git a/chromium/chrome/renderer/performance_manager/OWNERS b/chromium/chrome/renderer/performance_manager/OWNERS index 0a5168e1964..6ef4e6df1e5 100644 --- a/chromium/chrome/renderer/performance_manager/OWNERS +++ b/chromium/chrome/renderer/performance_manager/OWNERS @@ -1 +1 @@ -file://chrome/browser/performance_manager/OWNERS +file://components/performance_manager/OWNERS diff --git a/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.cc b/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.cc index 4410ede5cb6..d662fe9310c 100644 --- a/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.cc +++ b/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.cc @@ -15,7 +15,6 @@ #include "base/values.h" #include "chrome/common/buildflags.h" #include "chrome/common/chrome_features.h" -#include "chrome/common/prerender_messages.h" #include "chrome/common/render_messages.h" #include "chrome/grit/generated_resources.h" #include "chrome/grit/renderer_resources.h" @@ -24,6 +23,7 @@ #include "chrome/renderer/plugins/plugin_preroller.h" #include "chrome/renderer/plugins/plugin_uma.h" #include "components/content_settings/renderer/content_settings_agent_impl.h" +#include "components/prerender/common/prerender_messages.h" #include "components/strings/grit/components_strings.h" #include "content/public/common/content_switches.h" #include "content/public/common/untrustworthy_context_menu_params.h" @@ -212,9 +212,9 @@ void ChromePluginPlaceholder::UpdateFailure() { } void ChromePluginPlaceholder::OnSetPrerenderMode( - prerender::PrerenderMode mode, + prerender::mojom::PrerenderMode mode, const std::string& histogram_prefix) { - OnSetIsPrerendering(mode != prerender::NO_PRERENDER); + OnSetIsPrerendering(mode != prerender::mojom::PrerenderMode::kNoPrerender); } void ChromePluginPlaceholder::PluginListChanged() { diff --git a/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.h b/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.h index 738a37a2679..88c7e513e2a 100644 --- a/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.h +++ b/chromium/chrome/renderer/plugins/chrome_plugin_placeholder.h @@ -11,9 +11,9 @@ #include "base/macros.h" #include "chrome/common/buildflags.h" #include "chrome/common/plugin.mojom.h" -#include "chrome/common/prerender_types.h" #include "chrome/renderer/plugins/power_saver_info.h" #include "components/plugins/renderer/loadable_plugin_placeholder.h" +#include "components/prerender/common/prerender_types.mojom.h" #include "content/public/renderer/context_menu_client.h" #include "content/public/renderer/render_thread_observer.h" #include "mojo/public/cpp/bindings/pending_remote.h" @@ -87,7 +87,7 @@ class ChromePluginPlaceholder final void UpdateFailure() override; // IPC message handlers: - void OnSetPrerenderMode(prerender::PrerenderMode mode, + void OnSetPrerenderMode(prerender::mojom::PrerenderMode mode, const std::string& histogram_prefix); chrome::mojom::PluginStatus status_; diff --git a/chromium/chrome/renderer/prerender/prerender_helper.cc b/chromium/chrome/renderer/prerender/prerender_helper.cc index 2097753f052..92103602000 100644 --- a/chromium/chrome/renderer/prerender/prerender_helper.cc +++ b/chromium/chrome/renderer/prerender/prerender_helper.cc @@ -6,8 +6,8 @@ #include "base/metrics/field_trial.h" #include "base/metrics/histogram_macros.h" -#include "chrome/common/prerender_messages.h" #include "chrome/common/prerender_url_loader_throttle.h" +#include "components/prerender/common/prerender_messages.h" #include "content/public/renderer/document_state.h" #include "content/public/renderer/render_frame.h" #include "content/public/renderer/render_thread.h" @@ -19,14 +19,14 @@ namespace prerender { PrerenderHelper::PrerenderHelper(content::RenderFrame* render_frame, - PrerenderMode prerender_mode, + prerender::mojom::PrerenderMode prerender_mode, const std::string& histogram_prefix) : content::RenderFrameObserver(render_frame), content::RenderFrameObserverTracker<PrerenderHelper>(render_frame), prerender_mode_(prerender_mode), histogram_prefix_(histogram_prefix), start_time_(base::TimeTicks::Now()) { - DCHECK_NE(prerender_mode_, NO_PRERENDER); + DCHECK_NE(prerender_mode_, prerender::mojom::PrerenderMode::kNoPrerender); } PrerenderHelper::~PrerenderHelper() = default; @@ -43,7 +43,7 @@ std::unique_ptr<blink::URLLoaderThrottle> PrerenderHelper::MaybeCreateThrottle( if (!prerender_helper) return nullptr; - mojo::PendingRemote<chrome::mojom::PrerenderCanceler> canceler; + mojo::PendingRemote<prerender::mojom::PrerenderCanceler> canceler; render_frame->GetBrowserInterfaceBroker()->GetInterface( canceler.InitWithNewPipeAndPassReceiver()); @@ -56,23 +56,26 @@ std::unique_ptr<blink::URLLoaderThrottle> PrerenderHelper::MaybeCreateThrottle( // static. bool PrerenderHelper::IsPrerendering(const content::RenderFrame* render_frame) { - return PrerenderHelper::GetPrerenderMode(render_frame) != NO_PRERENDER; + return PrerenderHelper::GetPrerenderMode(render_frame) != + prerender::mojom::PrerenderMode::kNoPrerender; } // static. -PrerenderMode PrerenderHelper::GetPrerenderMode( +prerender::mojom::PrerenderMode PrerenderHelper::GetPrerenderMode( const content::RenderFrame* render_frame) { PrerenderHelper* helper = PrerenderHelper::Get(render_frame); if (!helper) - return NO_PRERENDER; + return prerender::mojom::PrerenderMode::kNoPrerender; - DCHECK_NE(helper->prerender_mode_, NO_PRERENDER); + DCHECK_NE(helper->prerender_mode_, + prerender::mojom::PrerenderMode::kNoPrerender); return helper->prerender_mode_; } void PrerenderHelper::DidFinishDocumentLoad() { - if (prerender_mode_ != PREFETCH_ONLY) + if (prerender_mode_ != prerender::mojom::PrerenderMode::kPrefetchOnly) return; + parsed_time_ = base::TimeTicks::Now(); prefetch_finished_ = true; if (prefetch_count_ == 0) @@ -93,11 +96,11 @@ void PrerenderHelper::OnDestruct() { delete this; } -void PrerenderHelper::OnSetIsPrerendering(PrerenderMode mode, +void PrerenderHelper::OnSetIsPrerendering(prerender::mojom::PrerenderMode mode, const std::string& histogram_prefix) { // Immediately after construction, |this| may receive the message that // triggered its creation. If so, ignore it. - if (mode != prerender::NO_PRERENDER) + if (mode != prerender::mojom::PrerenderMode::kNoPrerender) return; std::vector<base::WeakPtr<PrerenderURLLoaderThrottle>> throttles = @@ -120,7 +123,7 @@ void PrerenderHelper::AddThrottle( // sending the "prefetch finished" signal until they are destroyed. This is // important since that signal tells the browser that it can tear down this // renderer which could interrupt subresource prefetching. - if (prerender_mode_ == PREFETCH_ONLY) { + if (prerender_mode_ == prerender::mojom::PrerenderMode::kPrefetchOnly) { prefetch_count_++; throttle->set_destruction_closure(base::BindOnce( &PrerenderHelper::OnThrottleDestroyed, weak_factory_.GetWeakPtr())); diff --git a/chromium/chrome/renderer/prerender/prerender_helper.h b/chromium/chrome/renderer/prerender/prerender_helper.h index 6bd8cbca918..e479e1a82cb 100644 --- a/chromium/chrome/renderer/prerender/prerender_helper.h +++ b/chromium/chrome/renderer/prerender/prerender_helper.h @@ -9,7 +9,7 @@ #include "base/macros.h" #include "base/memory/weak_ptr.h" #include "base/time/time.h" -#include "chrome/common/prerender_types.h" +#include "components/prerender/common/prerender_types.mojom.h" #include "content/public/renderer/render_frame_observer.h" #include "content/public/renderer/render_frame_observer_tracker.h" @@ -27,7 +27,7 @@ class PrerenderHelper public content::RenderFrameObserverTracker<PrerenderHelper> { public: PrerenderHelper(content::RenderFrame* render_frame, - PrerenderMode prerender_mode, + prerender::mojom::PrerenderMode prerender_mode, const std::string& histogram_prefix); ~PrerenderHelper() override; @@ -40,10 +40,12 @@ class PrerenderHelper // Returns true if |render_frame| is currently prerendering. static bool IsPrerendering(const content::RenderFrame* render_frame); - static PrerenderMode GetPrerenderMode( + static prerender::mojom::PrerenderMode GetPrerenderMode( const content::RenderFrame* render_frame); - PrerenderMode prerender_mode() const { return prerender_mode_; } + prerender::mojom::PrerenderMode prerender_mode() const { + return prerender_mode_; + } std::string histogram_prefix() const { return histogram_prefix_; } private: @@ -52,14 +54,14 @@ class PrerenderHelper bool OnMessageReceived(const IPC::Message& message) override; void OnDestruct() override; - void OnSetIsPrerendering(PrerenderMode mode, + void OnSetIsPrerendering(prerender::mojom::PrerenderMode mode, const std::string& histogram_prefix); void AddThrottle(const base::WeakPtr<PrerenderURLLoaderThrottle>& throttle); void OnThrottleDestroyed(); void SendPrefetchFinished(); - PrerenderMode prerender_mode_; + prerender::mojom::PrerenderMode prerender_mode_; std::string histogram_prefix_; // Pending requests for this frame.. diff --git a/chromium/chrome/renderer/prerender/prerenderer_client.cc b/chromium/chrome/renderer/prerender/prerenderer_client.cc index c82d6e61174..76c809faacf 100644 --- a/chromium/chrome/renderer/prerender/prerenderer_client.cc +++ b/chromium/chrome/renderer/prerender/prerenderer_client.cc @@ -23,7 +23,8 @@ PrerendererClient::~PrerendererClient() = default; bool PrerendererClient::IsPrefetchOnly() { return PrerenderHelper::GetPrerenderMode( - render_view()->GetMainRenderFrame()) == PREFETCH_ONLY; + render_view()->GetMainRenderFrame()) == + prerender::mojom::PrerenderMode::kPrefetchOnly; } void PrerendererClient::OnDestruct() { diff --git a/chromium/chrome/renderer/previews/resource_loading_hints_agent.cc b/chromium/chrome/renderer/previews/resource_loading_hints_agent.cc index 5e0737dc806..154008b2061 100644 --- a/chromium/chrome/renderer/previews/resource_loading_hints_agent.cc +++ b/chromium/chrome/renderer/previews/resource_loading_hints_agent.cc @@ -7,6 +7,7 @@ #include <vector> #include "base/metrics/histogram_macros.h" +#include "base/metrics/histogram_macros_local.h" #include "content/public/renderer/render_frame.h" #include "third_party/blink/public/platform/web_loading_hints_provider.h" #include "third_party/blink/public/platform/web_string.h" @@ -38,8 +39,6 @@ ResourceLoadingHintsAgent::ResourceLoadingHintsAgent( content::RenderFrameObserverTracker<ResourceLoadingHintsAgent>( render_frame) { DCHECK(render_frame); - DCHECK(IsMainFrame()); - associated_interfaces->AddInterface(base::BindRepeating( &ResourceLoadingHintsAgent::SetReceiver, base::Unretained(this))); } @@ -51,17 +50,23 @@ GURL ResourceLoadingHintsAgent::GetDocumentURL() const { void ResourceLoadingHintsAgent::DidStartNavigation( const GURL& url, base::Optional<blink::WebNavigationType> navigation_type) { + if (!IsMainFrame()) + return; subresource_redirect_hints_agent_.DidStartNavigation(); } void ResourceLoadingHintsAgent::ReadyToCommitNavigation( blink::WebDocumentLoader* document_loader) { + if (!IsMainFrame()) + return; + subresource_redirect_hints_agent_.ReadyToCommitNavigation( render_frame()->GetRoutingID()); } void ResourceLoadingHintsAgent::DidCreateNewDocument() { - DCHECK(IsMainFrame()); + if (!IsMainFrame()) + return; if (!GetDocumentURL().SchemeIsHTTPOrHTTPS()) return; @@ -101,7 +106,8 @@ bool ResourceLoadingHintsAgent::IsMainFrame() const { void ResourceLoadingHintsAgent::SetResourceLoadingHints( blink::mojom::PreviewsResourceLoadingHintsPtr resource_loading_hints) { - DCHECK(IsMainFrame()); + if (!IsMainFrame()) + return; UMA_HISTOGRAM_COUNTS_100( "ResourceLoadingHints.CountBlockedSubresourcePatterns", @@ -117,8 +123,37 @@ void ResourceLoadingHintsAgent::SetResourceLoadingHints( void ResourceLoadingHintsAgent::SetCompressPublicImagesHints( blink::mojom::CompressPublicImagesHintsPtr images_hints) { + if (!IsMainFrame()) + return; subresource_redirect_hints_agent_.SetCompressPublicImagesHints( std::move(images_hints)); } +void ResourceLoadingHintsAgent::NotifyHttpsImageCompressionFetchFailed( + base::TimeDelta retry_after) { + if (!subresource_redirect_service_remote_) { + render_frame()->GetRemoteAssociatedInterfaces()->GetInterface( + &subresource_redirect_service_remote_); + } + subresource_redirect_service_remote_->NotifyCompressedImageFetchFailed( + retry_after); +} + +void ResourceLoadingHintsAgent::SetLiteVideoHint( + blink::mojom::LiteVideoHintPtr lite_video_hint) { + auto* lite_video_hint_agent = + lite_video::LiteVideoHintAgent::Get(render_frame()); + if (lite_video_hint_agent) + lite_video_hint_agent->SetLiteVideoHint(std::move(lite_video_hint)); +} + +void ResourceLoadingHintsAgent::StopThrottlingMediaRequests() { + auto* lite_video_hint_agent = + lite_video::LiteVideoHintAgent::Get(render_frame()); + if (lite_video_hint_agent) { + LOCAL_HISTOGRAM_BOOLEAN("LiteVideo.HintsAgent.StopThrottling", true); + lite_video_hint_agent->StopThrottling(); + } +} + } // namespace previews diff --git a/chromium/chrome/renderer/previews/resource_loading_hints_agent.h b/chromium/chrome/renderer/previews/resource_loading_hints_agent.h index 85fdc2b4d88..370b2789d54 100644 --- a/chromium/chrome/renderer/previews/resource_loading_hints_agent.h +++ b/chromium/chrome/renderer/previews/resource_loading_hints_agent.h @@ -8,9 +8,10 @@ #include <memory> #include "base/bind.h" -#include "base/logging.h" #include "base/macros.h" #include "base/optional.h" +#include "chrome/common/subresource_redirect_service.mojom.h" +#include "chrome/renderer/lite_video/lite_video_hint_agent.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h" #include "content/public/renderer/render_frame_observer.h" #include "content/public/renderer/render_frame_observer_tracker.h" @@ -44,6 +45,9 @@ class ResourceLoadingHintsAgent return subresource_redirect_hints_agent_; } + // Notifies the browser process that https image compression fetch had failed. + void NotifyHttpsImageCompressionFetchFailed(base::TimeDelta retry_after); + private: // content::RenderFrameObserver: void DidStartNavigation( @@ -61,6 +65,9 @@ class ResourceLoadingHintsAgent resource_loading_hints) override; void SetCompressPublicImagesHints( blink::mojom::CompressPublicImagesHintsPtr images_hints) override; + void SetLiteVideoHint( + blink::mojom::LiteVideoHintPtr lite_video_hint) override; + void StopThrottlingMediaRequests() override; void SetReceiver( mojo::PendingAssociatedReceiver< @@ -74,6 +81,10 @@ class ResourceLoadingHintsAgent mojo::AssociatedReceiver<blink::mojom::PreviewsResourceLoadingHintsReceiver> receiver_{this}; + mojo::AssociatedRemote< + subresource_redirect::mojom::SubresourceRedirectService> + subresource_redirect_service_remote_; + subresource_redirect::SubresourceRedirectHintsAgent subresource_redirect_hints_agent_; diff --git a/chromium/chrome/renderer/printing/chrome_print_render_frame_helper_delegate.cc b/chromium/chrome/renderer/printing/chrome_print_render_frame_helper_delegate.cc index 49086491846..8fd541ac8bf 100644 --- a/chromium/chrome/renderer/printing/chrome_print_render_frame_helper_delegate.cc +++ b/chromium/chrome/renderer/printing/chrome_print_render_frame_helper_delegate.cc @@ -43,10 +43,14 @@ blink::WebElement ChromePrintRenderFrameHelperDelegate::GetPdfElement( url.host_piece() == extension_misc::kPdfExtensionId; if (inside_print_preview || inside_pdf_extension) { // <object> with id="plugin" is created in - // chrome/browser/resources/pdf/pdf_viewer.js. - auto plugin_element = frame->GetDocument().GetElementById("plugin"); - if (!plugin_element.IsNull()) { - return plugin_element; + // chrome/browser/resources/pdf/pdf_viewer_base.js. + auto viewer_element = frame->GetDocument().GetElementById("viewer"); + if (!viewer_element.IsNull() && !viewer_element.ShadowRoot().IsNull()) { + auto plugin_element = + viewer_element.ShadowRoot().QuerySelector("#plugin"); + if (!plugin_element.IsNull()) { + return plugin_element; + } } NOTREACHED(); } diff --git a/chromium/chrome/renderer/resources/default_100_percent/common/sadplugin.png b/chromium/chrome/renderer/resources/default_100_percent/common/sadplugin.png Binary files differdeleted file mode 100644 index 9ad1225ac67..00000000000 --- a/chromium/chrome/renderer/resources/default_100_percent/common/sadplugin.png +++ /dev/null diff --git a/chromium/chrome/renderer/resources/default_100_percent/common/webview-crash.png b/chromium/chrome/renderer/resources/default_100_percent/common/webview-crash.png Binary files differdeleted file mode 100644 index 0ce1db18805..00000000000 --- a/chromium/chrome/renderer/resources/default_100_percent/common/webview-crash.png +++ /dev/null diff --git a/chromium/chrome/renderer/resources/default_200_percent/common/sadplugin.png b/chromium/chrome/renderer/resources/default_200_percent/common/sadplugin.png Binary files differdeleted file mode 100644 index 78286797176..00000000000 --- a/chromium/chrome/renderer/resources/default_200_percent/common/sadplugin.png +++ /dev/null diff --git a/chromium/chrome/renderer/resources/default_200_percent/common/webview-crash.png b/chromium/chrome/renderer/resources/default_200_percent/common/webview-crash.png Binary files differdeleted file mode 100644 index eeb34b253a6..00000000000 --- a/chromium/chrome/renderer/resources/default_200_percent/common/webview-crash.png +++ /dev/null diff --git a/chromium/chrome/renderer/resources/extensions/cast_streaming_receiver_session_custom_bindings.js b/chromium/chrome/renderer/resources/extensions/cast_streaming_receiver_session_custom_bindings.js deleted file mode 100644 index 64c5370d233..00000000000 --- a/chromium/chrome/renderer/resources/extensions/cast_streaming_receiver_session_custom_bindings.js +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Custom binding for the Cast Streaming Session API. - -var natives = requireNative('cast_streaming_natives'); - -apiBridge.registerCustomHook(function(bindingsAPI, extensionId) { - var apiFunctions = bindingsAPI.apiFunctions; - apiFunctions.setHandleRequest( - 'createAndBind', - function(ap, vp, local, weidgth, height, fr, url, cb, op) { - }); -}); diff --git a/chromium/chrome/renderer/resources/extensions/cast_streaming_rtp_stream_custom_bindings.js b/chromium/chrome/renderer/resources/extensions/cast_streaming_rtp_stream_custom_bindings.js deleted file mode 100644 index 8e4f312d147..00000000000 --- a/chromium/chrome/renderer/resources/extensions/cast_streaming_rtp_stream_custom_bindings.js +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Custom binding for the Cast Streaming RtpStream API. - -var natives = requireNative('cast_streaming_natives'); - -apiBridge.registerCustomHook(function(bindingsAPI, extensionId) { - var apiFunctions = bindingsAPI.apiFunctions; - - apiFunctions.setHandleRequest('destroy', - function(transportId) { - natives.DestroyCastRtpStream(transportId); - }); - apiFunctions.setHandleRequest('getSupportedParams', - function(transportId) { - return natives.GetSupportedParamsCastRtpStream(transportId); - }); - apiFunctions.setHandleRequest('start', - function(transportId, params) { - natives.StartCastRtpStream(transportId, params); - }); - apiFunctions.setHandleRequest('stop', - function(transportId) { - natives.StopCastRtpStream(transportId); - }); - apiFunctions.setHandleRequest('toggleLogging', - function(transportId, enable) { - natives.ToggleLogging(transportId, enable); - }); - apiFunctions.setHandleRequest('getRawEvents', - function(transportId, extraData, callback) { - natives.GetRawEvents(transportId, extraData, callback); - }); - apiFunctions.setHandleRequest('getStats', - function(transportId, callback) { - natives.GetStats(transportId, callback); - }); -}); diff --git a/chromium/chrome/renderer/resources/extensions/cast_streaming_session_custom_bindings.js b/chromium/chrome/renderer/resources/extensions/cast_streaming_session_custom_bindings.js deleted file mode 100644 index 05afdd6a0d9..00000000000 --- a/chromium/chrome/renderer/resources/extensions/cast_streaming_session_custom_bindings.js +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Custom binding for the Cast Streaming Session API. - -var natives = requireNative('cast_streaming_natives'); - -apiBridge.registerCustomHook(function(bindingsAPI, extensionId) { - var apiFunctions = bindingsAPI.apiFunctions; - apiFunctions.setHandleRequest('create', - function(audioTrack, videoTrack, callback) { - natives.CreateSession(audioTrack, videoTrack, callback); - }); -}); diff --git a/chromium/chrome/renderer/resources/extensions/cast_streaming_udp_transport_custom_bindings.js b/chromium/chrome/renderer/resources/extensions/cast_streaming_udp_transport_custom_bindings.js deleted file mode 100644 index c3f64736225..00000000000 --- a/chromium/chrome/renderer/resources/extensions/cast_streaming_udp_transport_custom_bindings.js +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2013 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Custom binding for the Cast Streaming UdpTransport API. - -var natives = requireNative('cast_streaming_natives'); - -apiBridge.registerCustomHook(function(bindingsAPI, extensionId) { - var apiFunctions = bindingsAPI.apiFunctions; - - apiFunctions.setHandleRequest('destroy', function(transportId) { - natives.DestroyCastUdpTransport(transportId); - }); - apiFunctions.setHandleRequest('setDestination', - function(transportId, destination) { - natives.SetDestinationCastUdpTransport(transportId, destination); - }); - apiFunctions.setHandleRequest('setOptions', - function(transportId, options) { - natives.SetOptionsCastUdpTransport(transportId, options); - }); -}); diff --git a/chromium/chrome/renderer/resources/extensions/chromeos_tts_stream_bindings.js b/chromium/chrome/renderer/resources/extensions/chromeos_tts_stream_bindings.js new file mode 100644 index 00000000000..de636071106 --- /dev/null +++ b/chromium/chrome/renderer/resources/extensions/chromeos_tts_stream_bindings.js @@ -0,0 +1,19 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +'use strict'; + +if ((typeof mojo === 'undefined') || !mojo.bindingsLibraryInitialized) { + loadScript('mojo_bindings'); +} +mojo.config.autoLoadMojomDeps = false; + +loadScript('chromeos.tts.mojom.tts_stream.mojom'); + +(function() { + let ptr = new chromeos.tts.mojom.TtsStreamPtr; + Mojo.bindInterface( + chromeos.tts.mojom.TtsStream.name, mojo.makeRequest(ptr).handle); + exports.$set('returnValue', ptr); +})(); diff --git a/chromium/chrome/renderer/resources/extensions/media_router_bindings.js b/chromium/chrome/renderer/resources/extensions/media_router_bindings.js index 6564c71d924..f24fc18793e 100644 --- a/chromium/chrome/renderer/resources/extensions/media_router_bindings.js +++ b/chromium/chrome/renderer/resources/extensions/media_router_bindings.js @@ -536,7 +536,7 @@ function routeToMojo_(route) { 'iconUrl': route.iconUrl, 'isLocal': route.isLocal, 'forDisplay': route.forDisplay, - 'isIncognito': route.offTheRecord, + 'isOffTheRecord': route.offTheRecord, 'isLocalPresentation': route.isOffscreenPresentation, 'controllerType': route.controllerType, 'presentationId': route.presentationId, @@ -973,6 +973,13 @@ MediaRouter.prototype.getMediaSinkServiceStatus = function() { } /** + * @return {!Promise<!{status: string}>} + */ +MediaRouter.prototype.getLogsAsString = function() { + return this.service_.getLogsAsString(); +} + +/** * @param {int32} target_tab_id * @param {!mojo.InterfaceRequest} request */ @@ -1186,19 +1193,19 @@ MediaRouteProvider.prototype.stopObservingMediaSinks = * @param {!number} tabId ID of tab requesting presentation. * @param {!mojo_base.mojom.TimeDelta} timeout If positive, the timeout * duration for the request. Otherwise, the default duration will be used. - * @param {!boolean} incognito If true, the route is being requested by - * an incognito profile. + * @param {!boolean} off_the_record If true, the route is being requested by + * an off_the_record profile. * @return {!Promise.<!Object>} A Promise resolving to an object describing * the newly created media route, or rejecting with an error message on * failure. */ MediaRouteProvider.prototype.createRoute = function(sourceUrn, sinkId, presentationId, origin, tabId, - timeout, incognito) { + timeout, off_the_record) { this.handlers_.onBeforeInvokeHandler(); return this.handlers_.createRoute( sourceUrn, sinkId, presentationId, origin, tabId, - Math.floor(timeout.microseconds / 1000), incognito) + Math.floor(timeout.microseconds / 1000), off_the_record) .then(function(route) { return toSuccessRouteResponse_(route); }, @@ -1217,19 +1224,19 @@ MediaRouteProvider.prototype.createRoute = * @param {!number} tabId ID of tab requesting join. * @param {!mojo_base.mojom.TimeDelta} timeout If positive, the timeout * duration for the request. Otherwise, the default duration will be used. - * @param {!boolean} incognito If true, the route is being requested by - * an incognito profile. + * @param {!boolean} off_the_record If true, the route is being requested by + * an off_the_record profile. * @return {!Promise.<!Object>} A Promise resolving to an object describing * the newly created media route, or rejecting with an error message on * failure. */ MediaRouteProvider.prototype.joinRoute = function(sourceUrn, presentationId, origin, tabId, timeout, - incognito) { + off_the_record) { this.handlers_.onBeforeInvokeHandler(); return this.handlers_.joinRoute( sourceUrn, presentationId, origin, tabId, - Math.floor(timeout.microseconds / 1000), incognito) + Math.floor(timeout.microseconds / 1000), off_the_record) .then(function(route) { return toSuccessRouteResponse_(route); }, @@ -1249,19 +1256,19 @@ MediaRouteProvider.prototype.joinRoute = * @param {!number} tabId ID of tab requesting join. * @param {!mojo_base.mojom.TimeDelta} timeout If positive, the timeout * duration for the request. Otherwise, the default duration will be used. - * @param {!boolean} incognito If true, the route is being requested by - * an incognito profile. + * @param {!boolean} off_the_record If true, the route is being requested by + * an off_the_record profile. * @return {!Promise.<!Object>} A Promise resolving to an object describing * the newly created media route, or rejecting with an error message on * failure. */ MediaRouteProvider.prototype.connectRouteByRouteId = function(sourceUrn, routeId, presentationId, origin, tabId, - timeout, incognito) { + timeout, off_the_record) { this.handlers_.onBeforeInvokeHandler(); return this.handlers_.connectRouteByRouteId( sourceUrn, routeId, presentationId, origin, tabId, - Math.floor(timeout.microseconds / 1000), incognito) + Math.floor(timeout.microseconds / 1000), off_the_record) .then(function(route) { return toSuccessRouteResponse_(route); }, diff --git a/chromium/chrome/renderer/resources/extensions/platform_keys/get_public_key.js b/chromium/chrome/renderer/resources/extensions/platform_keys/get_public_key.js index 5bf7255e88b..d5105f0902f 100644 --- a/chromium/chrome/renderer/resources/extensions/platform_keys/get_public_key.js +++ b/chromium/chrome/renderer/resources/extensions/platform_keys/get_public_key.js @@ -10,12 +10,14 @@ var normalizeAlgorithm = // Returns the normalized parameters of |importParams|. // Any unknown parameters will be ignored. function normalizeImportParams(importParams) { - if (!importParams.name || - Object.prototype.toString.call(importParams.name) !== '[object String]') { + if (!importParams.name || typeof importParams.name !== 'string') { throw new Error('Algorithm: name: Missing or not a String'); } - var filteredParams = { name: importParams.name }; + var filteredParams = { + name: importParams.name, + namedCurve: importParams.namedCurve + }; var hashIsNone = false; if (importParams.hash) { @@ -29,9 +31,13 @@ function normalizeImportParams(importParams) { } } + if (importParams.name === 'ECDSA' && importParams.namedCurve !== 'P-256') { + throw new Error('Only P-256 named curve is supported.'); + } + // Apply WebCrypto's algorithm normalization. var resultParams = normalizeAlgorithm(filteredParams, 'ImportKey'); - if (!resultParams ) { + if (!resultParams) { throw new Error('A required parameter was missing or out-of-range'); } if (hashIsNone) { @@ -47,11 +53,14 @@ function combineAlgorithms(algorithm, importParams) { algorithm.publicExponent = new Uint8Array(algorithm.publicExponent); } - algorithm.hash = importParams.hash; + if (importParams.hash) { + algorithm.hash = importParams.hash; + } return algorithm; } function getPublicKey(cert, importParams, callback) { + // TODO(crbug.com/1096486): Check cert type is ArrayBuffer. importParams = normalizeImportParams(importParams); internalAPI.getPublicKey( cert, importParams.name, function(publicKey, algorithm) { @@ -64,4 +73,21 @@ function getPublicKey(cert, importParams, callback) { }); } +function getPublicKeyBySpki(publicKeySpkiDer, importParams, callback) { + if (!(publicKeySpkiDer instanceof ArrayBuffer)){ + throw $Error.self('publicKeySpkiDer: Not an ArrayBuffer'); + } + importParams = normalizeImportParams(importParams); + internalAPI.getPublicKeyBySpki( + publicKeySpkiDer, importParams.name, function(publicKey, algorithm) { + if (bindingUtil.hasLastError()) { + callback(); + return; + } + var combinedAlgorithm = combineAlgorithms(algorithm, importParams); + callback(publicKey, combinedAlgorithm); + }); +} + exports.$set('getPublicKey', getPublicKey); +exports.$set('getPublicKeyBySpki', getPublicKeyBySpki); diff --git a/chromium/chrome/renderer/resources/extensions/platform_keys/internal_api.js b/chromium/chrome/renderer/resources/extensions/platform_keys/internal_api.js index b3c4c0fef36..324eee7fa77 100644 --- a/chromium/chrome/renderer/resources/extensions/platform_keys/internal_api.js +++ b/chromium/chrome/renderer/resources/extensions/platform_keys/internal_api.js @@ -2,8 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(crbug.com/1096029): Make consumers use the API directly by calling +// getInternalApi themselves. var binding = getInternalApi('platformKeysInternal'); exports.$set('selectClientCertificates', binding.selectClientCertificates); exports.$set('sign', binding.sign); exports.$set('getPublicKey', binding.getPublicKey); +exports.$set('getPublicKeyBySpki', binding.getPublicKeyBySpki); diff --git a/chromium/chrome/renderer/resources/extensions/platform_keys_custom_bindings.js b/chromium/chrome/renderer/resources/extensions/platform_keys_custom_bindings.js index 70faf9bd0a8..147269975db 100644 --- a/chromium/chrome/renderer/resources/extensions/platform_keys_custom_bindings.js +++ b/chromium/chrome/renderer/resources/extensions/platform_keys_custom_bindings.js @@ -5,7 +5,9 @@ // Custom binding for the platformKeys API. var SubtleCrypto = require('platformKeys.SubtleCrypto').SubtleCrypto; -var getPublicKey = require('platformKeys.getPublicKey').getPublicKey; +var publicKeyUtil = require('platformKeys.getPublicKeyUtil'); +var getPublicKey = publicKeyUtil.getPublicKey; +var getPublicKeyBySpki = publicKeyUtil.getPublicKeyBySpki; var internalAPI = require('platformKeys.internalAPI'); var keyModule = require('platformKeys.Key'); @@ -60,4 +62,18 @@ apiBridge.registerCustomHook(function(api) { createPrivateKey(publicKey, algorithm)); }); }); + + apiFunctions.setHandleRequest( + 'getKeyPairBySpki', function(publicKeySpkiDer, params, callback) { + getPublicKeyBySpki( + publicKeySpkiDer, params, function(publicKey, algorithm) { + if (bindingUtil.hasLastError()) { + callback(); + return; + } + callback( + createPublicKey(publicKey, algorithm), + createPrivateKey(publicKey, algorithm)); + }); + }); }); diff --git a/chromium/chrome/renderer/resources/renderer_resources.grd b/chromium/chrome/renderer/resources/renderer_resources.grd index 3f31e156ca7..076fd011a55 100644 --- a/chromium/chrome/renderer/resources/renderer_resources.grd +++ b/chromium/chrome/renderer/resources/renderer_resources.grd @@ -9,10 +9,6 @@ <output filename="renderer_resources_300_percent.pak" type="data_package" context="default_300_percent" /> </outputs> <release seq="1"> - <structures fallback_to_low_resolution="true"> - <structure type="chrome_scaled_image" name="IDR_SAD_WEBVIEW" file="common\webview-crash.png" /> - <structure type="chrome_scaled_image" name="IDR_SAD_PLUGIN" file="common\sadplugin.png" /> - </structures> <includes> <include name="IDR_BLOCKED_PLUGIN_HTML" file="plugins/blocked_plugin.html" flattenhtml="true" type="BINDATA" /> <include name="IDR_DISABLED_PLUGIN_HTML" file="plugins/disabled_plugin.html" flattenhtml="true" type="BINDATA" /> @@ -25,10 +21,6 @@ <!-- Custom bindings for extension APIs. --> <include name="IDR_ACTION_CUSTOM_BINDINGS_JS" file="extensions\action_custom_bindings.js" type="BINDATA" /> <include name="IDR_BROWSER_ACTION_CUSTOM_BINDINGS_JS" file="extensions\browser_action_custom_bindings.js" type="BINDATA" /> - <include name="IDR_CAST_STREAMING_RTP_STREAM_CUSTOM_BINDINGS_JS" file="extensions\cast_streaming_rtp_stream_custom_bindings.js" type="BINDATA" /> - <include name="IDR_CAST_STREAMING_SESSION_CUSTOM_BINDINGS_JS" file="extensions\cast_streaming_session_custom_bindings.js" type="BINDATA" /> - <include name="IDR_CAST_STREAMING_UDP_TRANSPORT_CUSTOM_BINDINGS_JS" file="extensions\cast_streaming_udp_transport_custom_bindings.js" type="BINDATA" /> - <include name="IDR_CAST_STREAMING_RECEIVER_SESSION_CUSTOM_BINDINGS_JS" file="extensions\cast_streaming_receiver_session_custom_bindings.js" type="BINDATA" /> <include name="IDR_CHROME_WEB_VIEW_INTERNAL_CUSTOM_BINDINGS_JS" file="extensions\web_view\chrome_web_view_internal_custom_bindings.js" type="BINDATA" /> <include name="IDR_CHROME_WEB_VIEW_JS" file="extensions\web_view\chrome_web_view.js" type="BINDATA" /> <include name="IDR_DECLARATIVE_CONTENT_CUSTOM_BINDINGS_JS" file="extensions\declarative_content_custom_bindings.js" type="BINDATA" /> @@ -72,6 +64,9 @@ <!-- ChromeOS IME Mojo service and bindings. --> <include name="IDR_IME_SERVICE_BINDINGS_JS" file="extensions\chromeos_ime_service_bindings.js" type="BINDATA" /> <include name="IDR_IME_SERVICE_MOJOM_JS" file="${mojom_root}\chromeos/services/ime/public/mojom/input_engine.mojom.js" use_base_dir="false" type="BINDATA" /> + + <include name="IDR_TTS_STREAM_BINDINGS_JS" file="extensions\chromeos_tts_stream_bindings.js" type="BINDATA" /> + <include name="IDR_TTS_STREAM_MOJOM_JS" file="${mojom_root}\chromeos/services/tts/public/mojom/tts_service.mojom.js" use_base_dir="false" type="BINDATA" /> </if> <!-- Media Router Mojo service and bindings. --> <include name="IDR_MEDIA_CONTROLLER_MOJOM_JS" file="${mojom_root}\chrome\common\media_router\mojom\media_controller.mojom.js" use_base_dir="false" type="BINDATA" /> diff --git a/chromium/chrome/renderer/safe_browsing/DEPS b/chromium/chrome/renderer/safe_browsing/DEPS index 76dae5d8e5a..6b8537549c7 100644 --- a/chromium/chrome/renderer/safe_browsing/DEPS +++ b/chromium/chrome/renderer/safe_browsing/DEPS @@ -1,9 +1,12 @@ include_rules = [ + "+components/paint_preview/common", "+components/safe_browsing/content/renderer", + "+components/safe_browsing/content/password_protection/visual_utils.h", "+components/safe_browsing/core/common", "+components/safe_browsing/core/proto/csd.pb.h", "+components/safe_browsing/core/proto/client_model.pb.h", "+components/safe_browsing/core/features.h", + "+cc/paint", "+third_party/smhasher", ] diff --git a/chromium/chrome/renderer/safe_browsing/feature_extractor_clock.cc b/chromium/chrome/renderer/safe_browsing/feature_extractor_clock.cc deleted file mode 100644 index 45fbd4d4437..00000000000 --- a/chromium/chrome/renderer/safe_browsing/feature_extractor_clock.cc +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/safe_browsing/feature_extractor_clock.h" - -namespace safe_browsing { - -FeatureExtractorClock::~FeatureExtractorClock() {} - -base::TimeTicks FeatureExtractorClock::Now() { - return base::TimeTicks::Now(); -} - -} // namespace safe_browsing diff --git a/chromium/chrome/renderer/safe_browsing/feature_extractor_clock.h b/chromium/chrome/renderer/safe_browsing/feature_extractor_clock.h deleted file mode 100644 index 930c1a4b7ab..00000000000 --- a/chromium/chrome/renderer/safe_browsing/feature_extractor_clock.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// A simple abstraction for getting the current time during feature extraction. -// This can be mocked out for testing. - -#ifndef CHROME_RENDERER_SAFE_BROWSING_FEATURE_EXTRACTOR_CLOCK_H_ -#define CHROME_RENDERER_SAFE_BROWSING_FEATURE_EXTRACTOR_CLOCK_H_ - -#include "base/macros.h" -#include "base/time/time.h" - -namespace safe_browsing { - -class FeatureExtractorClock { - public: - FeatureExtractorClock() {} - virtual ~FeatureExtractorClock(); - - // Returns the current time. May be mocked for testing. - virtual base::TimeTicks Now(); - - private: - DISALLOW_COPY_AND_ASSIGN(FeatureExtractorClock); -}; - -} // namespace safe_browsing - -#endif // CHROME_RENDERER_SAFE_BROWSING_FEATURE_EXTRACTOR_CLOCK_H_ diff --git a/chromium/chrome/renderer/safe_browsing/mock_feature_extractor_clock.cc b/chromium/chrome/renderer/safe_browsing/mock_feature_extractor_clock.cc deleted file mode 100644 index 665baf0b5c3..00000000000 --- a/chromium/chrome/renderer/safe_browsing/mock_feature_extractor_clock.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2011 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "chrome/renderer/safe_browsing/mock_feature_extractor_clock.h" - -namespace safe_browsing { - -MockFeatureExtractorClock::MockFeatureExtractorClock() - : FeatureExtractorClock() {} - -MockFeatureExtractorClock::~MockFeatureExtractorClock() {} - -} // namespace safe_browsing diff --git a/chromium/chrome/renderer/safe_browsing/mock_feature_extractor_clock.h b/chromium/chrome/renderer/safe_browsing/mock_feature_extractor_clock.h deleted file mode 100644 index 7012ee81090..00000000000 --- a/chromium/chrome/renderer/safe_browsing/mock_feature_extractor_clock.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2010 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. -// -// A mock implementation of FeatureExtractorClock for testing. - -#ifndef CHROME_RENDERER_SAFE_BROWSING_MOCK_FEATURE_EXTRACTOR_CLOCK_H_ -#define CHROME_RENDERER_SAFE_BROWSING_MOCK_FEATURE_EXTRACTOR_CLOCK_H_ - -#include "base/macros.h" -#include "chrome/renderer/safe_browsing/feature_extractor_clock.h" -#include "testing/gmock/include/gmock/gmock.h" - -namespace safe_browsing { - -class MockFeatureExtractorClock : public FeatureExtractorClock { - public: - MockFeatureExtractorClock(); - ~MockFeatureExtractorClock() override; - - MOCK_METHOD0(Now, base::TimeTicks()); - - private: - DISALLOW_COPY_AND_ASSIGN(MockFeatureExtractorClock); -}; - -} // namespace safe_browsing - -#endif // CHROME_RENDERER_SAFE_BROWSING_MOCK_FEATURE_EXTRACTOR_CLOCK_H_ diff --git a/chromium/chrome/renderer/safe_browsing/phishing_classifier.cc b/chromium/chrome/renderer/safe_browsing/phishing_classifier.cc index 0842c7ea50a..c53c18d31b5 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_classifier.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_classifier.cc @@ -11,18 +11,18 @@ #include "base/callback.h" #include "base/compiler_specific.h" #include "base/location.h" -#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "base/single_thread_task_runner.h" #include "base/strings/string_util.h" #include "base/threading/thread_task_runner_handle.h" +#include "cc/paint/skia_paint_canvas.h" #include "chrome/common/url_constants.h" -#include "chrome/renderer/safe_browsing/feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/features.h" #include "chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h" #include "chrome/renderer/safe_browsing/phishing_term_feature_extractor.h" #include "chrome/renderer/safe_browsing/phishing_url_feature_extractor.h" #include "chrome/renderer/safe_browsing/scorer.h" +#include "components/paint_preview/common/paint_preview_tracker.h" #include "components/safe_browsing/core/proto/csd.pb.h" #include "content/public/renderer/render_frame.h" #include "crypto/sha2.h" @@ -32,6 +32,7 @@ #include "third_party/blink/public/web/web_document_loader.h" #include "third_party/blink/public/web/web_local_frame.h" #include "third_party/blink/public/web/web_view.h" +#include "ui/gfx/geometry/rect_conversions.h" #include "url/gurl.h" namespace safe_browsing { @@ -39,32 +40,29 @@ namespace safe_browsing { const float PhishingClassifier::kInvalidScore = -1.0; const float PhishingClassifier::kPhishyThreshold = 0.5; -PhishingClassifier::PhishingClassifier(content::RenderFrame* render_frame, - FeatureExtractorClock* clock) - : render_frame_(render_frame), scorer_(nullptr), clock_(clock) { +PhishingClassifier::PhishingClassifier(content::RenderFrame* render_frame) + : render_frame_(render_frame), scorer_(nullptr) { Clear(); } PhishingClassifier::~PhishingClassifier() { // The RenderView should have called CancelPendingClassification() before // we are destroyed. - CheckNoPendingClassification(); + DCHECK(done_callback_.is_null()); + DCHECK(!page_text_); } void PhishingClassifier::set_phishing_scorer(const Scorer* scorer) { - CheckNoPendingClassification(); + DCHECK(done_callback_.is_null()); + DCHECK(!page_text_); scorer_ = scorer; if (scorer_) { - url_extractor_.reset(new PhishingUrlFeatureExtractor); - dom_extractor_.reset(new PhishingDOMFeatureExtractor(clock_.get())); - term_extractor_.reset(new PhishingTermFeatureExtractor( - &scorer_->page_terms(), - &scorer_->page_words(), - scorer_->max_words_per_term(), - scorer_->murmurhash3_seed(), - scorer_->max_shingles_per_page(), - scorer_->shingle_size(), - clock_.get())); + url_extractor_ = std::make_unique<PhishingUrlFeatureExtractor>(); + dom_extractor_ = std::make_unique<PhishingDOMFeatureExtractor>(); + term_extractor_ = std::make_unique<PhishingTermFeatureExtractor>( + &scorer_->page_terms(), &scorer_->page_words(), + scorer_->max_words_per_term(), scorer_->murmurhash3_seed(), + scorer_->max_shingles_per_page(), scorer_->shingle_size()); } else { // We're disabling client-side phishing detection, so tear down all // of the relevant objects. @@ -75,7 +73,7 @@ void PhishingClassifier::set_phishing_scorer(const Scorer* scorer) { } bool PhishingClassifier::is_ready() const { - return scorer_ != NULL; + return !!scorer_; } void PhishingClassifier::BeginClassification(const base::string16* page_text, @@ -84,7 +82,8 @@ void PhishingClassifier::BeginClassification(const base::string16* page_text, // The RenderView should have called CancelPendingClassification() before // starting a new classification, so DCHECK this. - CheckNoPendingClassification(); + DCHECK(done_callback_.is_null()); + DCHECK(!page_text_); // However, in an opt build, we will go ahead and clean up the pending // classification so that we can start in a known state. CancelPendingClassification(); @@ -158,41 +157,70 @@ void PhishingClassifier::DOMExtractionFinished(bool success) { void PhishingClassifier::TermExtractionFinished(bool success) { if (success) { - blink::WebLocalFrame* main_frame = render_frame_->GetWebFrame(); - - // Hash all of the features so that they match the model, then compute - // the score. - FeatureMap hashed_features; - ClientPhishingRequest verdict; - verdict.set_model_version(scorer_->model_version()); - verdict.set_url(main_frame->GetDocument().Url().GetString().Utf8()); - for (const auto& it : features_->features()) { - bool result = hashed_features.AddRealFeature( - crypto::SHA256HashString(it.first), it.second); - DCHECK(result); - ClientPhishingRequest::Feature* feature = verdict.add_feature_map(); - feature->set_name(it.first); - feature->set_value(it.second); - } - for (const auto& it : *shingle_hashes_) { - verdict.add_shingle_hashes(it); - } - float score = static_cast<float>(scorer_->ComputeScore(hashed_features)); - verdict.set_client_score(score); - verdict.set_is_phishing(score >= scorer_->threshold_probability()); - RunCallback(verdict); + ExtractVisualFeatures(); } else { RunFailureCallback(); } } -void PhishingClassifier::CheckNoPendingClassification() { - DCHECK(done_callback_.is_null()); - DCHECK(!page_text_); - if (!done_callback_.is_null() || page_text_) { - LOG(ERROR) << "Classification in progress, missing call to " - << "CancelPendingClassification"; +void PhishingClassifier::ExtractVisualFeatures() { + blink::WebLocalFrame* frame = render_frame_->GetWebFrame(); + gfx::SizeF viewport_size = frame->View()->VisualViewportSize(); + gfx::Rect bounds = ToEnclosingRect(gfx::RectF(viewport_size)); + bitmap_ = std::make_unique<SkBitmap>(); + // Use the Rec. 2020 color space, in case the user input is wide-gamut. + sk_sp<SkColorSpace> rec2020 = SkColorSpace::MakeRGB( + {2.22222f, 0.909672f, 0.0903276f, 0.222222f, 0.0812429f, 0, 0}, + SkNamedGamut::kRec2020); + SkImageInfo bitmap_info = SkImageInfo::Make( + bounds.width(), bounds.height(), SkColorType::kRGBA_8888_SkColorType, + SkAlphaType::kUnpremul_SkAlphaType, rec2020); + if (!bitmap_->tryAllocPixels(bitmap_info)) + return VisualExtractionFinished(/*success=*/false); + SkCanvas sk_canvas(*bitmap_); + cc::SkiaPaintCanvas cc_canvas(&sk_canvas); + auto tracker = std::make_unique<paint_preview::PaintPreviewTracker>( + base::UnguessableToken::Create(), frame->GetEmbeddingToken(), + /*is_main_frame=*/true); + cc_canvas.SetPaintPreviewTracker(tracker.get()); + VisualExtractionFinished(frame->CapturePaintPreview( + bounds, &cc_canvas, /*include_linked_destinations=*/false)); +} + +void PhishingClassifier::VisualExtractionFinished(bool success) { + if (!success) { + RunFailureCallback(); + return; + } + + blink::WebLocalFrame* main_frame = render_frame_->GetWebFrame(); + + // Hash all of the features so that they match the model, then compute + // the score. + FeatureMap hashed_features; + ClientPhishingRequest verdict; + verdict.set_model_version(scorer_->model_version()); + verdict.set_url(main_frame->GetDocument().Url().GetString().Utf8()); + for (const auto& it : features_->features()) { + bool result = hashed_features.AddRealFeature( + crypto::SHA256HashString(it.first), it.second); + DCHECK(result); + ClientPhishingRequest::Feature* feature = verdict.add_feature_map(); + feature->set_name(it.first); + feature->set_value(it.second); } + for (const auto& it : *shingle_hashes_) { + verdict.add_shingle_hashes(it); + } + float score = static_cast<float>(scorer_->ComputeScore(hashed_features)); + verdict.set_client_score(score); + verdict.set_is_phishing(score >= scorer_->threshold_probability()); + + if (scorer_->GetMatchingVisualTargets(*bitmap_, &verdict)) { + verdict.set_is_phishing(true); + } + + RunCallback(verdict); } void PhishingClassifier::RunCallback(const ClientPhishingRequest& verdict) { @@ -211,10 +239,11 @@ void PhishingClassifier::RunFailureCallback() { } void PhishingClassifier::Clear() { - page_text_ = NULL; + page_text_ = nullptr; done_callback_.Reset(); - features_.reset(NULL); - shingle_hashes_.reset(NULL); + features_.reset(nullptr); + shingle_hashes_.reset(nullptr); + bitmap_.reset(nullptr); } } // namespace safe_browsing diff --git a/chromium/chrome/renderer/safe_browsing/phishing_classifier.h b/chromium/chrome/renderer/safe_browsing/phishing_classifier.h index 9528c434cce..401ae402dba 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_classifier.h +++ b/chromium/chrome/renderer/safe_browsing/phishing_classifier.h @@ -27,6 +27,7 @@ #include "base/macros.h" #include "base/memory/weak_ptr.h" #include "base/strings/string16.h" +#include "third_party/skia/include/core/SkBitmap.h" namespace content { class RenderFrame; @@ -34,7 +35,6 @@ class RenderFrame; namespace safe_browsing { class ClientPhishingRequest; -class FeatureExtractorClock; class FeatureMap; class PhishingDOMFeatureExtractor; class PhishingTermFeatureExtractor; @@ -55,11 +55,9 @@ class PhishingClassifier { static const float kInvalidScore; // Creates a new PhishingClassifier object that will operate on - // |render_view|. |clock| is used to time feature extractor operations, and - // the PhishingClassifier takes ownership of this object. Note that the - // classifier will not be 'ready' until set_phishing_scorer() is called. - PhishingClassifier(content::RenderFrame* render_frame, - FeatureExtractorClock* clock); + // |render_view|. Note that the classifier will not be 'ready' until + // set_phishing_scorer() is called. + explicit PhishingClassifier(content::RenderFrame* render_frame); virtual ~PhishingClassifier(); // Sets a scorer for the classifier to use in computing the phishiness score. @@ -110,15 +108,18 @@ class PhishingClassifier { void DOMExtractionFinished(bool success); // Callback to be run when term feature extraction is complete. + // If it was successful, begins visual feature extraction, otherwise runs the + // DoneCallback with a non-phishy verdict. + void TermExtractionFinished(bool success); + + // Called to extract the visual features of the current page. + void ExtractVisualFeatures(); + + // Callback when visual feature extraction is complete. // If it was successful, computes a score and runs the DoneCallback. // If extraction was unsuccessful, runs the DoneCallback with a // non-phishy verdict. - void TermExtractionFinished(bool success); - - // Helper to verify that there is no pending phishing classification. Dies - // in debug builds if the state is not as expected. This is a no-op in - // release builds. - void CheckNoPendingClassification(); + void VisualExtractionFinished(bool success); // Helper method to run the DoneCallback and clear the state. void RunCallback(const ClientPhishingRequest& verdict); @@ -132,7 +133,6 @@ class PhishingClassifier { content::RenderFrame* render_frame_; // owns us const Scorer* scorer_; // owned by the caller - std::unique_ptr<FeatureExtractorClock> clock_; std::unique_ptr<PhishingUrlFeatureExtractor> url_extractor_; std::unique_ptr<PhishingDOMFeatureExtractor> dom_extractor_; std::unique_ptr<PhishingTermFeatureExtractor> term_extractor_; @@ -141,6 +141,7 @@ class PhishingClassifier { std::unique_ptr<FeatureMap> features_; std::unique_ptr<std::set<uint32_t>> shingle_hashes_; const base::string16* page_text_; // owned by the caller + std::unique_ptr<SkBitmap> bitmap_; DoneCallback done_callback_; // Used in scheduling BeginFeatureExtraction tasks. diff --git a/chromium/chrome/renderer/safe_browsing/phishing_classifier_browsertest.cc b/chromium/chrome/renderer/safe_browsing/phishing_classifier_browsertest.cc index 2600269b203..484958e30db 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_classifier_browsertest.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_classifier_browsertest.cc @@ -12,9 +12,9 @@ #include "base/run_loop.h" #include "base/strings/string16.h" #include "base/strings/utf_string_conversions.h" +#include "base/test/test_discardable_memory_allocator.h" #include "chrome/renderer/chrome_content_renderer_client.h" #include "chrome/renderer/safe_browsing/features.h" -#include "chrome/renderer/safe_browsing/mock_feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/murmurhash3_util.h" #include "chrome/renderer/safe_browsing/scorer.h" #include "chrome/test/base/chrome_render_view_test.h" @@ -60,6 +60,8 @@ class PhishingClassifierTest : public ChromeRenderViewTest { ChromeRenderViewTest::SetUp(); PrepareModel(); SetUpClassifier(); + + base::DiscardableMemoryAllocator::SetInstance(&test_allocator_); } void PrepareModel() { @@ -99,18 +101,16 @@ class PhishingClassifierTest : public ChromeRenderViewTest { model.set_max_shingles_per_page(100); model.set_shingle_size(3); - clock_ = new MockFeatureExtractorClock; + // Add an empty visual target to ensure visual detection runs. + model.mutable_vision_model()->add_targets(); + scorer_.reset(Scorer::Create(model.SerializeAsString())); ASSERT_TRUE(scorer_.get()); - - // These tests don't exercise the extraction timing. - EXPECT_CALL(*clock_, Now()) - .WillRepeatedly(::testing::Return(base::TimeTicks::Now())); } void SetUpClassifier() { - classifier_.reset( - new PhishingClassifier(view_->GetMainRenderFrame(), clock_)); + classifier_ = + std::make_unique<PhishingClassifier>(view_->GetMainRenderFrame()); // No scorer yet, so the classifier is not ready. ASSERT_FALSE(classifier_->is_ready()); @@ -138,6 +138,8 @@ class PhishingClassifierTest : public ChromeRenderViewTest { verdict.feature_map(i).value()); } is_phishing_ = verdict.is_phishing(); + screenshot_phash_ = verdict.screenshot_phash(); + phash_dimension_size_ = verdict.phash_dimension_size(); } void LoadHtml(const GURL& url, const std::string& content) { @@ -153,7 +155,6 @@ class PhishingClassifierTest : public ChromeRenderViewTest { std::string response_content_; std::unique_ptr<Scorer> scorer_; std::unique_ptr<PhishingClassifier> classifier_; - MockFeatureExtractorClock* clock_; // Owned by classifier_. // Features that are in the model. const std::string url_tld_token_net_; @@ -165,6 +166,11 @@ class PhishingClassifierTest : public ChromeRenderViewTest { FeatureMap feature_map_; float phishy_score_; bool is_phishing_; + std::string screenshot_phash_; + int phash_dimension_size_; + + // A DiscardableMemoryAllocator is needed for certain Skia operations. + base::TestDiscardableMemoryAllocator test_allocator_; }; TEST_F(PhishingClassifierTest, TestClassificationOfPhishingDotComHttp) { @@ -261,6 +267,15 @@ TEST_F(PhishingClassifierTest, DisableDetection) { EXPECT_FALSE(classifier_->is_ready()); } +TEST_F(PhishingClassifierTest, TestSendsVisualHash) { + LoadHtml(GURL("https://host.net"), + "<html><body><a href=\"http://safe.com/\">login</a></body></html>"); + RunPhishingClassifier(&page_text_); + + EXPECT_EQ(phash_dimension_size_, 48); + EXPECT_FALSE(screenshot_phash_.empty()); +} + // TODO(jialiul): Add test to verify that classification only starts on GET // method. It seems there is no easy way to simulate a HTTP POST in // ChromeRenderViewTest. diff --git a/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.cc b/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.cc index 0afae4ea172..b65c048fd82 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.cc @@ -12,10 +12,8 @@ #include "base/callback.h" #include "base/debug/stack_trace.h" #include "base/lazy_instance.h" -#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "base/no_destructor.h" -#include "chrome/renderer/safe_browsing/feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/phishing_classifier.h" #include "chrome/renderer/safe_browsing/scorer.h" #include "components/safe_browsing/content/common/safe_browsing.mojom-forward.h" @@ -54,42 +52,6 @@ base::LazyInstance<std::unique_ptr<const safe_browsing::Scorer>>:: } // namespace -// static -void PhishingClassifierFilter::Create( - mojo::PendingReceiver<mojom::PhishingModelSetter> receiver) { - mojo::MakeSelfOwnedReceiver(std::make_unique<PhishingClassifierFilter>(), - std::move(receiver)); -} - -PhishingClassifierFilter::PhishingClassifierFilter() {} - -PhishingClassifierFilter::~PhishingClassifierFilter() {} - -void PhishingClassifierFilter::SetPhishingModel(const std::string& model) { - safe_browsing::Scorer* scorer = NULL; - // An empty model string means we should disable client-side phishing - // detection. - if (!model.empty()) { - scorer = safe_browsing::Scorer::Create(model); - if (!scorer) { - DLOG(ERROR) << "Unable to create a PhishingScorer - corrupt model?"; - return; - } - } - for (auto* delegate : PhishingClassifierDelegates()) - delegate->SetPhishingScorer(scorer); - g_phishing_scorer.Get().reset(scorer); -} - -// static -PhishingClassifierDelegate* PhishingClassifierDelegate::Create( - content::RenderFrame* render_frame, - PhishingClassifier* classifier) { - // Private constructor and public static Create() method to facilitate - // stubbing out this class for binary-size reduction purposes. - return new PhishingClassifierDelegate(render_frame, classifier); -} - PhishingClassifierDelegate::PhishingClassifierDelegate( content::RenderFrame* render_frame, PhishingClassifier* classifier) @@ -99,8 +61,7 @@ PhishingClassifierDelegate::PhishingClassifierDelegate( is_classifying_(false) { PhishingClassifierDelegates().insert(this); if (!classifier) { - classifier = - new PhishingClassifier(render_frame, new FeatureExtractorClock()); + classifier = new PhishingClassifier(render_frame); } classifier_.reset(classifier); @@ -118,6 +79,29 @@ PhishingClassifierDelegate::~PhishingClassifierDelegate() { PhishingClassifierDelegates().erase(this); } +void PhishingClassifierDelegate::SetPhishingModel(const std::string& model) { + safe_browsing::Scorer* scorer = nullptr; + // An empty model string means we should disable client-side phishing + // detection. + if (!model.empty()) { + scorer = safe_browsing::Scorer::Create(model); + if (!scorer) + return; + } + for (auto* delegate : PhishingClassifierDelegates()) + delegate->SetPhishingScorer(scorer); + g_phishing_scorer.Get().reset(scorer); +} + +// static +PhishingClassifierDelegate* PhishingClassifierDelegate::Create( + content::RenderFrame* render_frame, + PhishingClassifier* classifier) { + // Private constructor and public static Create() method to facilitate + // stubbing out this class for binary-size reduction purposes. + return new PhishingClassifierDelegate(render_frame, classifier); +} + void PhishingClassifierDelegate::SetPhishingScorer( const safe_browsing::Scorer* scorer) { if (is_classifying_) { @@ -159,23 +143,22 @@ void PhishingClassifierDelegate::StartPhishingDetection( } void PhishingClassifierDelegate::DidCommitProvisionalLoad( - bool is_same_document_navigation, ui::PageTransition transition) { blink::WebLocalFrame* frame = render_frame()->GetWebFrame(); // A new page is starting to load, so cancel classificaiton. - // + CancelPendingClassification(NAVIGATE_AWAY); + if (!frame->Parent()) + last_main_frame_transition_ = transition; +} + +void PhishingClassifierDelegate::DidFinishSameDocumentNavigation() { // TODO(bryner): We shouldn't need to cancel classification if the navigation // is within the same document. However, if we let classification continue in // this case, we need to properly deal with the fact that PageCaptured will // be called again for the same-document navigation. We need to be sure not // to swap out the page text while the term feature extractor is still // running. - CancelPendingClassification(is_same_document_navigation ? NAVIGATE_WITHIN_PAGE - : NAVIGATE_AWAY); - if (frame->Parent()) - return; - - last_main_frame_transition_ = transition; + CancelPendingClassification(NAVIGATE_WITHIN_PAGE); } void PhishingClassifierDelegate::PageCaptured(base::string16* page_text, @@ -196,8 +179,8 @@ void PhishingClassifierDelegate::PageCaptured(base::string16* page_text, have_page_text_ = true; GURL stripped_last_load_url(StripRef(last_finished_load_url_)); + // Check if toplevel URL has changed. if (stripped_last_load_url == StripRef(last_url_sent_to_classifier_)) { - DVLOG(2) << "Toplevel URL is unchanged, not starting classification."; return; } @@ -225,8 +208,6 @@ void PhishingClassifierDelegate::CancelPendingClassification( void PhishingClassifierDelegate::ClassificationDone( const ClientPhishingRequest& verdict) { - DVLOG(2) << "Phishy verdict = " << verdict.is_phishing() - << " score = " << verdict.client_score(); is_phishing_detection_running_ = false; if (callback_.is_null()) return; @@ -255,7 +236,6 @@ void PhishingClassifierDelegate::MaybeStartClassification() { // classified at all (as opposed to deferring it until we get an IPC or // the load completes), we discard the page text since it won't be needed. if (!classifier_->is_ready()) { - DVLOG(2) << "Not starting classification, no Scorer created."; is_phishing_detection_running_ = false; // Keep classifier_page_text_, in case a Scorer is set later. if (!callback_.is_null()) @@ -268,7 +248,6 @@ void PhishingClassifierDelegate::MaybeStartClassification() { // Skip loads from session history navigation. However, update the // last URL sent to the classifier, so that we'll properly detect // same-document navigations. - DVLOG(2) << "Not starting classification for back/forward navigation"; last_url_sent_to_classifier_ = last_finished_load_url_; classifier_page_text_.clear(); // we won't need this. have_page_text_ = false; @@ -281,7 +260,6 @@ void PhishingClassifierDelegate::MaybeStartClassification() { GURL stripped_last_load_url(StripRef(last_finished_load_url_)); if (!have_page_text_) { - DVLOG(2) << "Not starting classification, there is no page text ready."; RecordEvent(SBPhishingClassifierEvent::kPageTextNotLoaded); return; } @@ -292,15 +270,11 @@ void PhishingClassifierDelegate::MaybeStartClassification() { // so defer classification for now. Note: the ref does not affect // any of the browser's preclassification checks, so we don't require it // to match. - DVLOG(2) << "Not starting classification, last url from browser is " - << last_url_received_from_browser_ << ", last finished load is " - << last_finished_load_url_; // Keep classifier_page_text_, in case the browser notifies us later that // we should classify the URL. return; } - DVLOG(2) << "Starting classification for " << last_finished_load_url_; last_url_sent_to_classifier_ = last_finished_load_url_; is_classifying_ = true; classifier_->BeginClassification( diff --git a/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.h b/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.h index 9910fc79509..80aa1193e95 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.h +++ b/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate.h @@ -38,21 +38,6 @@ enum class SBPhishingClassifierEvent { kMaxValue = kDestructedBeforeClassificationDone, }; -class PhishingClassifierFilter : public mojom::PhishingModelSetter { - public: - PhishingClassifierFilter(); - ~PhishingClassifierFilter() override; - - static void Create( - mojo::PendingReceiver<mojom::PhishingModelSetter> receiver); - - private: - // mojom::PhishingModelSetter - void SetPhishingModel(const std::string& model) override; - - DISALLOW_COPY_AND_ASSIGN(PhishingClassifierFilter); -}; - class PhishingClassifierDelegate : public content::RenderFrameObserver, public mojom::PhishingDetector { public: @@ -63,6 +48,9 @@ class PhishingClassifierDelegate : public content::RenderFrameObserver, PhishingClassifier* classifier); ~PhishingClassifierDelegate() override; + // mojom::PhishingDetector + void SetPhishingModel(const std::string& model) override; + // Called by the RenderFrame once there is a phishing scorer available. // The scorer is passed on to the classifier. void SetPhishingScorer(const safe_browsing::Scorer* scorer); @@ -78,10 +66,11 @@ class PhishingClassifierDelegate : public content::RenderFrameObserver, // Called by the RenderFrame when a page has started loading in the given // WebFrame. Typically, this will cause any pending classification to be - // cancelled. However, if the navigation is within the same page, we - // continue running the current classification. - void DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) override; + // cancelled. + void DidCommitProvisionalLoad(ui::PageTransition transition) override; + // Called by the RenderFrame when the same-document navigation has been + // committed. We continue running the current classification. + void DidFinishSameDocumentNavigation() override; private: friend class PhishingClassifierDelegateTest; diff --git a/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate_browsertest.cc b/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate_browsertest.cc index 2acec9d451c..0ae43da87c5 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate_browsertest.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_classifier_delegate_browsertest.cc @@ -37,7 +37,7 @@ namespace { class MockPhishingClassifier : public PhishingClassifier { public: explicit MockPhishingClassifier(content::RenderFrame* render_frame) - : PhishingClassifier(render_frame, NULL /* clock */) {} + : PhishingClassifier(render_frame) {} ~MockPhishingClassifier() override {} @@ -229,6 +229,25 @@ TEST_F(PhishingClassifierDelegateTest, Navigation) { EXPECT_CALL(*classifier_, CancelPendingClassification()); } +TEST_F(PhishingClassifierDelegateTest, NoPhishingModel) { + ASSERT_FALSE(classifier_->is_ready()); + delegate_->SetPhishingModel(""); + // The scorer is nullptr so the classifier should still not be ready. + ASSERT_FALSE(classifier_->is_ready()); +} + +TEST_F(PhishingClassifierDelegateTest, HasPhishingModel) { + ASSERT_FALSE(classifier_->is_ready()); + + ClientSideModel model; + model.set_max_words_per_term(1); + delegate_->SetPhishingModel(model.SerializeAsString()); + ASSERT_TRUE(classifier_->is_ready()); + + // The delegate will cancel pending classification on destruction. + EXPECT_CALL(*classifier_, CancelPendingClassification()); +} + TEST_F(PhishingClassifierDelegateTest, NoScorer) { // For this test, we'll create the delegate with no scorer available yet. ASSERT_FALSE(classifier_->is_ready()); diff --git a/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.cc b/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.cc index f1a9fb37d82..c31a97320cb 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.cc @@ -9,13 +9,12 @@ #include "base/bind.h" #include "base/compiler_specific.h" #include "base/location.h" -#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "base/single_thread_task_runner.h" #include "base/strings/string_util.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/default_tick_clock.h" #include "base/time/time.h" -#include "chrome/renderer/safe_browsing/feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/features.h" #include "content/public/renderer/render_view.h" #include "net/base/registry_controlled_domains/registry_controlled_domain.h" @@ -102,16 +101,17 @@ struct PhishingDOMFeatureExtractor::FrameData { std::string domain; }; -PhishingDOMFeatureExtractor::PhishingDOMFeatureExtractor( - FeatureExtractorClock* clock) - : clock_(clock) { +PhishingDOMFeatureExtractor::PhishingDOMFeatureExtractor() + : clock_(base::DefaultTickClock::GetInstance()) { Clear(); } PhishingDOMFeatureExtractor::~PhishingDOMFeatureExtractor() { // The RenderView should have called CancelPendingExtraction() before // we are destroyed. - CheckNoPendingExtraction(); + DCHECK(done_callback_.is_null()); + DCHECK(!cur_frame_data_.get()); + DCHECK(cur_document_.IsNull()); } void PhishingDOMFeatureExtractor::ExtractFeatures(blink::WebDocument document, @@ -119,7 +119,9 @@ void PhishingDOMFeatureExtractor::ExtractFeatures(blink::WebDocument document, DoneCallback done_callback) { // The RenderView should have called CancelPendingExtraction() before // starting a new extraction, so DCHECK this. - CheckNoPendingExtraction(); + DCHECK(done_callback_.is_null()); + DCHECK(!cur_frame_data_.get()); + DCHECK(cur_document_.IsNull()); // However, in an opt build, we will go ahead and clean up the pending // extraction so that we can start in a known state. CancelPendingExtraction(); @@ -127,7 +129,7 @@ void PhishingDOMFeatureExtractor::ExtractFeatures(blink::WebDocument document, features_ = features; done_callback_ = std::move(done_callback); - page_feature_state_.reset(new PageFeatureState(clock_->Now())); + page_feature_state_ = std::make_unique<PageFeatureState>(clock_->NowTicks()); cur_document_ = document; base::ThreadTaskRunnerHandle::Get()->PostTask( @@ -145,7 +147,7 @@ void PhishingDOMFeatureExtractor::CancelPendingExtraction() { void PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout() { DCHECK(page_feature_state_.get()); ++page_feature_state_->num_iterations; - base::TimeTicks current_chunk_start_time = clock_->Now(); + base::TimeTicks current_chunk_start_time = clock_->NowTicks(); if (cur_document_.IsNull()) { // This will only happen if we weren't able to get the document for the @@ -166,7 +168,7 @@ void PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout() { // modified between our chunks of work. Log how long this takes, so we // can tell if it's too slow. UMA_HISTOGRAM_TIMES("SBClientPhishing.DOMFeatureResumeTime", - clock_->Now() - current_chunk_start_time); + clock_->NowTicks() - current_chunk_start_time); } else { // We just moved to a new frame, so update our frame state // and advance to the first element. @@ -190,10 +192,9 @@ void PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout() { if (++num_elements >= kClockCheckGranularity) { num_elements = 0; - base::TimeTicks now = clock_->Now(); + base::TimeTicks now = clock_->NowTicks(); if (now - page_feature_state_->start_time >= base::TimeDelta::FromMilliseconds(kMaxTotalTimeMs)) { - DLOG(ERROR) << "Feature extraction took too long, giving up"; // We expect this to happen infrequently, so record when it does. UMA_HISTOGRAM_COUNTS_1M("SBClientPhishing.DOMFeatureTimeout", 1); RunCallback(false); @@ -233,20 +234,16 @@ void PhishingDOMFeatureExtractor::ExtractFeaturesWithTimeout() { void PhishingDOMFeatureExtractor::HandleLink( const blink::WebElement& element) { // Count the number of times we link to a different host. - if (!element.HasAttribute("href")) { - DVLOG(1) << "Skipping anchor tag with no href"; + if (!element.HasAttribute("href")) return; - } // Retrieve the link and resolve the link in case it's relative. blink::WebURL full_url = CompleteURL(element, element.GetAttribute("href")); std::string domain; bool is_external = IsExternalDomain(full_url, &domain); - if (domain.empty()) { - DVLOG(1) << "Could not extract domain from link: " << full_url; + if (domain.empty()) return; - } if (is_external) { ++page_feature_state_->external_links; @@ -279,10 +276,8 @@ void PhishingDOMFeatureExtractor::HandleForm( std::string domain; bool is_external = IsExternalDomain(full_url, &domain); - if (domain.empty()) { - DVLOG(1) << "Could not extract domain from form action: " << full_url; + if (domain.empty()) return; - } if (is_external) { ++page_feature_state_->action_other_domain; @@ -292,22 +287,16 @@ void PhishingDOMFeatureExtractor::HandleForm( void PhishingDOMFeatureExtractor::HandleImage( const blink::WebElement& element) { - if (!element.HasAttribute("src")) { - DVLOG(1) << "Skipping img tag with no src"; - } - // Record whether the image points to a different domain. blink::WebURL full_url = CompleteURL(element, element.GetAttribute("src")); std::string domain; bool is_external = IsExternalDomain(full_url, &domain); - if (domain.empty()) { - DVLOG(1) << "Could not extract domain from image src: " << full_url; + if (domain.empty()) return; - } - if (is_external) { + if (is_external) ++page_feature_state_->img_other_domain; - } + ++page_feature_state_->total_imgs; } @@ -340,17 +329,6 @@ void PhishingDOMFeatureExtractor::HandleScript( ++page_feature_state_->num_script_tags; } -void PhishingDOMFeatureExtractor::CheckNoPendingExtraction() { - DCHECK(done_callback_.is_null()); - DCHECK(!cur_frame_data_.get()); - DCHECK(cur_document_.IsNull()); - if (!done_callback_.is_null() || cur_frame_data_.get() || - !cur_document_.IsNull()) { - LOG(ERROR) << "Extraction in progress, missing call to " - << "CancelPendingExtraction"; - } -} - void PhishingDOMFeatureExtractor::RunCallback(bool success) { // Record some timing stats that we can use to evaluate feature extraction // performance. These include both successful and failed extractions. @@ -358,7 +336,7 @@ void PhishingDOMFeatureExtractor::RunCallback(bool success) { UMA_HISTOGRAM_COUNTS_1M("SBClientPhishing.DOMFeatureIterations", page_feature_state_->num_iterations); UMA_HISTOGRAM_TIMES("SBClientPhishing.DOMFeatureTotalTime", - clock_->Now() - page_feature_state_->start_time); + clock_->NowTicks() - page_feature_state_->start_time); DCHECK(!done_callback_.is_null()); std::move(done_callback_).Run(success); @@ -366,9 +344,9 @@ void PhishingDOMFeatureExtractor::RunCallback(bool success) { } void PhishingDOMFeatureExtractor::Clear() { - features_ = NULL; + features_ = nullptr; done_callback_.Reset(); - cur_frame_data_.reset(NULL); + cur_frame_data_.reset(nullptr); cur_document_.Reset(); } diff --git a/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h b/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h index 6f4e2c1503c..4639f620f2c 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h +++ b/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor.h @@ -16,6 +16,7 @@ #include "base/callback.h" #include "base/macros.h" #include "base/memory/weak_ptr.h" +#include "base/time/tick_clock.h" #include "third_party/blink/public/web/web_document.h" class GURL; @@ -25,7 +26,6 @@ class WebElement; } namespace safe_browsing { -class FeatureExtractorClock; class FeatureMap; class PhishingDOMFeatureExtractor { @@ -35,9 +35,7 @@ class PhishingDOMFeatureExtractor { typedef base::OnceCallback<void(bool)> DoneCallback; // Creates a PhishingDOMFeatureExtractor instance. - // |clock| is used for timing feature extractor operations, and may be - // mocked for testing. The caller maintains ownership of the clock. - explicit PhishingDOMFeatureExtractor(FeatureExtractorClock* clock); + PhishingDOMFeatureExtractor(); virtual ~PhishingDOMFeatureExtractor(); // Begins extracting features into the given FeatureMap for the page. @@ -55,6 +53,8 @@ class PhishingDOMFeatureExtractor { // is unloaded or the PhishingDOMFeatureExtractor is destroyed. void CancelPendingExtraction(); + void SetTickClockForTesting(const base::TickClock* clock) { clock_ = clock; } + private: struct FrameData; struct PageFeatureState; @@ -88,11 +88,6 @@ class PhishingDOMFeatureExtractor { void HandleInput(const blink::WebElement& element); void HandleScript(const blink::WebElement& element); - // Helper to verify that there is no pending feature extraction. Dies in - // debug builds if the state is not as expected. This is a no-op in release - // builds. - void CheckNoPendingExtraction(); - // Runs |done_callback_| and then clears all internal state. void RunCallback(bool success); @@ -122,9 +117,7 @@ class PhishingDOMFeatureExtractor { // description of which features are computed. void InsertFeatures(); - - // Non-owned pointer to our clock. - FeatureExtractorClock* clock_; + const base::TickClock* clock_; // The output parameters from the most recent call to ExtractFeatures(). FeatureMap* features_; // The caller keeps ownership of this. diff --git a/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor_browsertest.cc b/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor_browsertest.cc index e64fd770af3..0586f5a9a5a 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor_browsertest.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_dom_feature_extractor_browsertest.cc @@ -13,7 +13,6 @@ #include "base/time/time.h" #include "chrome/renderer/chrome_content_renderer_client.h" #include "chrome/renderer/safe_browsing/features.h" -#include "chrome/renderer/safe_browsing/mock_feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/test_utils.h" #include "chrome/test/base/chrome_render_view_test.h" #include "content/public/common/content_switches.h" @@ -30,22 +29,28 @@ #include "third_party/blink/public/web/web_script_source.h" #include "ui/native_theme/native_theme_features.h" +using blink::WebRuntimeFeatures; using ::testing::DoAll; using ::testing::Invoke; using ::testing::Return; -using blink::WebRuntimeFeatures; +using ::testing::StrictMock; namespace safe_browsing { +class MockTickClock : public base::TickClock { + public: + MockTickClock() = default; + ~MockTickClock() override = default; + + MOCK_CONST_METHOD0(NowTicks, base::TimeTicks()); +}; + // TestPhishingDOMFeatureExtractor has nearly identical behavior as // PhishingDOMFeatureExtractor, except the IsExternalDomain() and // CompleteURL() functions. This is to work around the fact that // ChromeRenderViewTest object does not know where the html content is hosted. class TestPhishingDOMFeatureExtractor : public PhishingDOMFeatureExtractor { public: - explicit TestPhishingDOMFeatureExtractor(FeatureExtractorClock* clock) - : PhishingDOMFeatureExtractor(clock) {} - void SetDocumentDomain(std::string domain) { base_domain_ = domain; } void SetURLToFrameDomainCheckingMap( @@ -106,9 +111,6 @@ class TestPhishingDOMFeatureExtractor : public PhishingDOMFeatureExtractor { const std::string frame_domain = it->second; full_url = GURL("http://" + it->second).Resolve(partial_url.Utf8()); url_to_frame_domain_map_[full_url.spec()] = it->second; - } else { - NOTREACHED() << "Testing input setup is incorrect. " - "Please check url_to_frame_domain_map_ setup."; } } return blink::WebURL(full_url); @@ -191,7 +193,7 @@ class PhishingDOMFeatureExtractorTest : public ChromeRenderViewTest { ChromeRenderViewTest::SetUp(); WebRuntimeFeatures::EnableOverlayScrollbars( ui::IsOverlayScrollbarEnabled()); - extractor_.reset(new TestPhishingDOMFeatureExtractor(&clock_)); + extractor_ = std::make_unique<TestPhishingDOMFeatureExtractor>(); } void TearDown() override { @@ -218,7 +220,6 @@ class PhishingDOMFeatureExtractorTest : public ChromeRenderViewTest { "document.body.removeChild(document.getElementById('frame1'));")); } - MockFeatureExtractorClock clock_; bool success_; std::unique_ptr<TestPhishingDOMFeatureExtractor> extractor_; scoped_refptr<content::MessageLoopRunner> message_loop_; @@ -226,9 +227,6 @@ class PhishingDOMFeatureExtractorTest : public ChromeRenderViewTest { }; TEST_F(PhishingDOMFeatureExtractorTest, FormFeatures) { - // This test doesn't exercise the extraction timing. - EXPECT_CALL(clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now())); - FeatureMap expected_features; expected_features.AddBooleanFeature(features::kPageHasForms); expected_features.AddRealFeature(features::kPageActionOtherDomainFreq, 0.25); @@ -296,9 +294,6 @@ TEST_F(PhishingDOMFeatureExtractorTest, FormFeatures) { } TEST_F(PhishingDOMFeatureExtractorTest, LinkFeatures) { - // This test doesn't exercise the extraction timing. - EXPECT_CALL(clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now())); - FeatureMap expected_features; expected_features.AddRealFeature(features::kPageExternalLinksFreq, 0.5); expected_features.AddRealFeature(features::kPageSecureLinksFreq, 0.0); @@ -334,9 +329,6 @@ TEST_F(PhishingDOMFeatureExtractorTest, LinkFeatures) { } TEST_F(PhishingDOMFeatureExtractorTest, ScriptAndImageFeatures) { - // This test doesn't exercise the extraction timing. - EXPECT_CALL(clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now())); - FeatureMap expected_features; expected_features.AddBooleanFeature(features::kPageNumScriptTagsGTOne); @@ -370,10 +362,6 @@ TEST_F(PhishingDOMFeatureExtractorTest, ScriptAndImageFeatures) { // iframe2 / \ iframe1 // \ iframe3 TEST_F(PhishingDOMFeatureExtractorTest, SubFrames) { - // This test doesn't exercise the extraction timing. - // Test that features are aggregated across all frames. - EXPECT_CALL(clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now())); - const char urlprefix[] = "data:text/html;charset=utf-8,"; std::unordered_map<std::string, std::string> url_iframe_map; std::string iframe1_nested_html( @@ -443,8 +431,7 @@ TEST_F(PhishingDOMFeatureExtractorTest, SubFrames) { } TEST_F(PhishingDOMFeatureExtractorTest, Continuation) { - // For this test, we'll cause the feature extraction to run multiple - // iterations by incrementing the clock. + StrictMock<MockTickClock> tick_clock; // This page has a total of 50 elements. For the external forms feature to // be computed correctly, the extractor has to examine the whole document. @@ -462,7 +449,7 @@ TEST_F(PhishingDOMFeatureExtractorTest, Continuation) { // Note that this assumes kClockCheckGranularity = 10 and // kMaxTimePerChunkMs = 10. base::TimeTicks now = base::TimeTicks::Now(); - EXPECT_CALL(clock_, Now()) + EXPECT_CALL(tick_clock, NowTicks()) // Time check at the start of extraction. .WillOnce(Return(now)) // Time check at the start of the first chunk of work. @@ -489,6 +476,7 @@ TEST_F(PhishingDOMFeatureExtractorTest, Continuation) { .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(54))) // A final time check for the histograms. .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(56))); + extractor_->SetTickClockForTesting(&tick_clock); FeatureMap expected_features; expected_features.AddBooleanFeature(features::kPageHasForms); @@ -502,13 +490,13 @@ TEST_F(PhishingDOMFeatureExtractorTest, Continuation) { ExtractFeatures("host.com", html, &features); ExpectFeatureMapsAreEqual(features, expected_features); // Make sure none of the mock expectations carry over to the next test. - ::testing::Mock::VerifyAndClearExpectations(&clock_); + ::testing::Mock::VerifyAndClearExpectations(&tick_clock); // Now repeat the test with the same page, but advance the clock faster so // that the extraction time exceeds the maximum total time for the feature // extractor. Extraction should fail. Note that this assumes // kMaxTotalTimeMs = 500. - EXPECT_CALL(clock_, Now()) + EXPECT_CALL(tick_clock, NowTicks()) // Time check at the start of extraction. .WillOnce(Return(now)) // Time check at the start of the first chunk of work. @@ -542,7 +530,8 @@ TEST_F(PhishingDOMFeatureExtractorTest, SubframeRemoval) { GURL iframe1_url(urlprefix + iframe1_html); base::TimeTicks now = base::TimeTicks::Now(); - EXPECT_CALL(clock_, Now()) + StrictMock<MockTickClock> tick_clock; + EXPECT_CALL(tick_clock, NowTicks()) // Time check at the start of extraction. .WillOnce(Return(now)) // Time check at the start of the first chunk of work. @@ -559,6 +548,7 @@ TEST_F(PhishingDOMFeatureExtractorTest, SubframeRemoval) { .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(27))) // A final time check for the histograms. .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(33))); + extractor_->SetTickClockForTesting(&tick_clock); FeatureMap expected_features; expected_features.AddBooleanFeature(features::kPageHasForms); diff --git a/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.cc b/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.cc index 2bea4a2ea43..61d045735cb 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.cc @@ -15,13 +15,12 @@ #include "base/i18n/break_iterator.h" #include "base/i18n/case_conversion.h" #include "base/location.h" -#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "base/single_thread_task_runner.h" #include "base/strings/utf_string_conversions.h" #include "base/threading/thread_task_runner_handle.h" +#include "base/time/default_tick_clock.h" #include "base/time/time.h" -#include "chrome/renderer/safe_browsing/feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/features.h" #include "chrome/renderer/safe_browsing/murmurhash3_util.h" #include "crypto/sha2.h" @@ -74,11 +73,8 @@ struct PhishingTermFeatureExtractor::ExtractionState { std::unique_ptr<base::i18n::BreakIterator> i(new base::i18n::BreakIterator( text, base::i18n::BreakIterator::BREAK_WORD)); - if (i->Init()) { + if (i->Init()) iterator = std::move(i); - } else { - DLOG(ERROR) << "failed to open iterator"; - } } }; @@ -88,22 +84,22 @@ PhishingTermFeatureExtractor::PhishingTermFeatureExtractor( size_t max_words_per_term, uint32_t murmurhash3_seed, size_t max_shingles_per_page, - size_t shingle_size, - FeatureExtractorClock* clock) + size_t shingle_size) : page_term_hashes_(page_term_hashes), page_word_hashes_(page_word_hashes), max_words_per_term_(max_words_per_term), murmurhash3_seed_(murmurhash3_seed), max_shingles_per_page_(max_shingles_per_page), shingle_size_(shingle_size), - clock_(clock) { + clock_(base::DefaultTickClock::GetInstance()) { Clear(); } PhishingTermFeatureExtractor::~PhishingTermFeatureExtractor() { // The RenderView should have called CancelPendingExtraction() before // we are destroyed. - CheckNoPendingExtraction(); + DCHECK(done_callback_.is_null()); + DCHECK(!state_.get()); } void PhishingTermFeatureExtractor::ExtractFeatures( @@ -113,7 +109,8 @@ void PhishingTermFeatureExtractor::ExtractFeatures( DoneCallback done_callback) { // The RenderView should have called CancelPendingExtraction() before // starting a new extraction, so DCHECK this. - CheckNoPendingExtraction(); + DCHECK(done_callback_.is_null()); + DCHECK(!state_.get()); // However, in an opt build, we will go ahead and clean up the pending // extraction so that we can start in a known state. CancelPendingExtraction(); @@ -122,7 +119,7 @@ void PhishingTermFeatureExtractor::ExtractFeatures( features_ = features; shingle_hashes_ = shingle_hashes, done_callback_ = std::move(done_callback); - state_.reset(new ExtractionState(*page_text_, clock_->Now())); + state_ = std::make_unique<ExtractionState>(*page_text_, clock_->NowTicks()); base::ThreadTaskRunnerHandle::Get()->PostTask( FROM_HERE, base::BindOnce(&PhishingTermFeatureExtractor::ExtractFeaturesWithTimeout, @@ -138,7 +135,7 @@ void PhishingTermFeatureExtractor::CancelPendingExtraction() { void PhishingTermFeatureExtractor::ExtractFeaturesWithTimeout() { DCHECK(state_.get()); ++state_->num_iterations; - base::TimeTicks current_chunk_start_time = clock_->Now(); + base::TimeTicks current_chunk_start_time = clock_->NowTicks(); if (!state_->iterator.get()) { // We failed to initialize the break iterator, so stop now. @@ -158,10 +155,9 @@ void PhishingTermFeatureExtractor::ExtractFeaturesWithTimeout() { if (num_words >= kClockCheckGranularity) { num_words = 0; - base::TimeTicks now = clock_->Now(); + base::TimeTicks now = clock_->NowTicks(); if (now - state_->start_time >= base::TimeDelta::FromMilliseconds(kMaxTotalTimeMs)) { - DLOG(ERROR) << "Feature extraction took too long, giving up"; // We expect this to happen infrequently, so record when it does. UMA_HISTOGRAM_COUNTS_1M("SBClientPhishing.TermFeatureTimeout", 1); RunCallback(false); @@ -261,15 +257,6 @@ void PhishingTermFeatureExtractor::HandleWord( } } -void PhishingTermFeatureExtractor::CheckNoPendingExtraction() { - DCHECK(done_callback_.is_null()); - DCHECK(!state_.get()); - if (!done_callback_.is_null() || state_.get()) { - LOG(ERROR) << "Extraction in progress, missing call to " - << "CancelPendingExtraction"; - } -} - void PhishingTermFeatureExtractor::RunCallback(bool success) { // Record some timing stats that we can use to evaluate feature extraction // performance. These include both successful and failed extractions. @@ -277,7 +264,7 @@ void PhishingTermFeatureExtractor::RunCallback(bool success) { UMA_HISTOGRAM_COUNTS_1M("SBClientPhishing.TermFeatureIterations", state_->num_iterations); UMA_HISTOGRAM_TIMES("SBClientPhishing.TermFeatureTotalTime", - clock_->Now() - state_->start_time); + clock_->NowTicks() - state_->start_time); DCHECK(!done_callback_.is_null()); std::move(done_callback_).Run(success); @@ -285,11 +272,11 @@ void PhishingTermFeatureExtractor::RunCallback(bool success) { } void PhishingTermFeatureExtractor::Clear() { - page_text_ = NULL; - features_ = NULL; - shingle_hashes_ = NULL; + page_text_ = nullptr; + features_ = nullptr; + shingle_hashes_ = nullptr; done_callback_.Reset(); - state_.reset(NULL); + state_.reset(nullptr); } } // namespace safe_browsing diff --git a/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.h b/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.h index d0cc48d7033..341f60704a2 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.h +++ b/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor.h @@ -29,9 +29,9 @@ #include "base/memory/weak_ptr.h" #include "base/strings/string16.h" #include "base/strings/string_piece.h" +#include "base/time/tick_clock.h" namespace safe_browsing { -class FeatureExtractorClock; class FeatureMap; class PhishingTermFeatureExtractor { @@ -63,8 +63,7 @@ class PhishingTermFeatureExtractor { size_t max_words_per_term, uint32_t murmurhash3_seed, size_t max_shingles_per_page, - size_t shingle_size, - FeatureExtractorClock* clock); + size_t shingle_size); ~PhishingTermFeatureExtractor(); // Begins extracting features from |page_text| into the given FeatureMap. @@ -90,6 +89,8 @@ class PhishingTermFeatureExtractor { // is unloaded or the PhishingTermFeatureExtractor is destroyed. void CancelPendingExtraction(); + void SetTickClockForTesting(const base::TickClock* clock) { clock_ = clock; } + private: struct ExtractionState; @@ -115,11 +116,6 @@ class PhishingTermFeatureExtractor { // Handles a single word in the page text. void HandleWord(const base::StringPiece16& word); - // Helper to verify that there is no pending feature extraction. Dies in - // debug builds if the state is not as expected. This is a no-op in release - // builds. - void CheckNoPendingExtraction(); - // Runs |done_callback_| and then clears all internal state. void RunCallback(bool success); @@ -149,7 +145,7 @@ class PhishingTermFeatureExtractor { const size_t shingle_size_; // Non-owned pointer to our clock. - FeatureExtractorClock* clock_; + const base::TickClock* clock_; // The output parameters from the most recent call to ExtractFeatures(). const base::string16* page_text_; // The caller keeps ownership of this. diff --git a/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor_unittest.cc b/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor_unittest.cc index c1b38303cbd..53fc9a643fb 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor_unittest.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_term_feature_extractor_unittest.cc @@ -24,7 +24,6 @@ #include "base/time/time.h" #include "build/build_config.h" #include "chrome/renderer/safe_browsing/features.h" -#include "chrome/renderer/safe_browsing/mock_feature_extractor_clock.h" #include "chrome/renderer/safe_browsing/murmurhash3_util.h" #include "chrome/renderer/safe_browsing/test_utils.h" #include "crypto/sha2.h" @@ -33,11 +32,20 @@ using base::ASCIIToUTF16; using ::testing::Return; +using ::testing::StrictMock; static const uint32_t kMurmurHash3Seed = 2777808611U; namespace safe_browsing { +class MockTickClock : public base::TickClock { + public: + MockTickClock() = default; + ~MockTickClock() override = default; + + MOCK_CONST_METHOD0(NowTicks, base::TimeTicks()); +}; + class PhishingTermFeatureExtractorTest : public ::testing::Test { protected: void SetUp() override { @@ -80,14 +88,9 @@ class PhishingTermFeatureExtractorTest : public ::testing::Test { } void ResetExtractor(size_t max_shingles_per_page) { - extractor_.reset(new PhishingTermFeatureExtractor( - &term_hashes_, - &word_hashes_, - 3 /* max_words_per_term */, - kMurmurHash3Seed, - max_shingles_per_page, - 4 /* shingle_size */, - &clock_)); + extractor_ = std::make_unique<PhishingTermFeatureExtractor>( + &term_hashes_, &word_hashes_, 3 /* max_words_per_term */, + kMurmurHash3Seed, max_shingles_per_page, 4 /* shingle_size */); } // Runs the TermFeatureExtractor on |page_text|, waiting for the @@ -133,7 +136,6 @@ class PhishingTermFeatureExtractorTest : public ::testing::Test { base::test::SingleThreadTaskEnvironment task_environment_; std::unique_ptr<base::RunLoop> active_run_loop_; - MockFeatureExtractorClock clock_; std::unique_ptr<PhishingTermFeatureExtractor> extractor_; std::unordered_set<std::string> term_hashes_; std::unordered_set<uint32_t> word_hashes_; @@ -141,9 +143,6 @@ class PhishingTermFeatureExtractorTest : public ::testing::Test { }; TEST_F(PhishingTermFeatureExtractorTest, ExtractFeatures) { - // This test doesn't exercise the extraction timing. - EXPECT_CALL(clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now())); - base::string16 page_text = ASCIIToUTF16("blah"); FeatureMap expected_features; // initially empty std::set<uint32_t> expected_shingle_hashes; @@ -306,7 +305,8 @@ TEST_F(PhishingTermFeatureExtractorTest, Continuation) { // Note that this assumes kClockCheckGranularity = 5 and // kMaxTimePerChunkMs = 10. base::TimeTicks now = base::TimeTicks::Now(); - EXPECT_CALL(clock_, Now()) + StrictMock<MockTickClock> tick_clock; + EXPECT_CALL(tick_clock, NowTicks()) // Time check at the start of extraction. .WillOnce(Return(now)) // Time check at the start of the first chunk of work. @@ -328,6 +328,7 @@ TEST_F(PhishingTermFeatureExtractorTest, Continuation) { .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(28))) // A final check for the histograms. .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(30))); + extractor_->SetTickClockForTesting(&tick_clock); FeatureMap expected_features; expected_features.AddBooleanFeature(features::kPageTerm + @@ -396,13 +397,13 @@ TEST_F(PhishingTermFeatureExtractorTest, Continuation) { ExpectFeatureMapsAreEqual(features, expected_features); EXPECT_THAT(expected_shingle_hashes, testing::ContainerEq(shingle_hashes)); // Make sure none of the mock expectations carry over to the next test. - ::testing::Mock::VerifyAndClearExpectations(&clock_); + ::testing::Mock::VerifyAndClearExpectations(&tick_clock); // Now repeat the test with the same text, but advance the clock faster so // that the extraction time exceeds the maximum total time for the feature // extractor. Extraction should fail. Note that this assumes // kMaxTotalTimeMs = 500. - EXPECT_CALL(clock_, Now()) + EXPECT_CALL(tick_clock, NowTicks()) // Time check at the start of extraction. .WillOnce(Return(now)) // Time check at the start of the first chunk of work. @@ -429,7 +430,8 @@ TEST_F(PhishingTermFeatureExtractorTest, PartialExtractionTest) { } base::TimeTicks now = base::TimeTicks::Now(); - EXPECT_CALL(clock_, Now()) + StrictMock<MockTickClock> tick_clock; + EXPECT_CALL(tick_clock, NowTicks()) // Time check at the start of extraction. .WillOnce(Return(now)) // Time check at the start of the first chunk of work. @@ -439,6 +441,7 @@ TEST_F(PhishingTermFeatureExtractorTest, PartialExtractionTest) { // Time check after the next 5 words. This should be greater than // kMaxTimePerChunkMs so that we stop and schedule extraction for later. .WillOnce(Return(now + base::TimeDelta::FromMilliseconds(14))); + extractor_->SetTickClockForTesting(&tick_clock); FeatureMap features; std::set<uint32_t> shingle_hashes; @@ -454,7 +457,8 @@ TEST_F(PhishingTermFeatureExtractorTest, PartialExtractionTest) { shingle_hashes.clear(); // This part doesn't exercise the extraction timing. - EXPECT_CALL(clock_, Now()).WillRepeatedly(Return(base::TimeTicks::Now())); + EXPECT_CALL(tick_clock, NowTicks()) + .WillRepeatedly(Return(base::TimeTicks::Now())); // Now extract normally and make sure nothing breaks. EXPECT_TRUE(ExtractFeatures(page_text.get(), &features, &shingle_hashes)); diff --git a/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor.cc b/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor.cc index 931c5203edd..403d1d060f7 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor.cc @@ -8,7 +8,6 @@ #include <string> #include <vector> -#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "base/strings/string_split.h" #include "base/strings/string_util.h" @@ -44,8 +43,8 @@ bool PhishingUrlFeatureExtractor::ExtractFeatures(const GURL& url, host, net::registry_controlled_domains::EXCLUDE_UNKNOWN_REGISTRIES, net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES); + // Check if TLD exists for host. if (registry_length == 0 || registry_length == std::string::npos) { - DVLOG(1) << "Could not find TLD for host: " << host; return false; } DCHECK_LT(registry_length, host.size()) << "Non-zero registry length, but " @@ -59,8 +58,8 @@ bool PhishingUrlFeatureExtractor::ExtractFeatures(const GURL& url, host.erase(tld_start - 1); std::vector<std::string> host_tokens = base::SplitString( host, ".", base::KEEP_WHITESPACE, base::SPLIT_WANT_NONEMPTY); + // Check if domain exists for host. if (host_tokens.empty()) { - DVLOG(1) << "Could not find domain for host: " << host; return false; } if (!features->AddBooleanFeature(features::kUrlDomainToken + diff --git a/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor_unittest.cc b/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor_unittest.cc index e5412a7bd4e..c87ab3daf88 100644 --- a/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor_unittest.cc +++ b/chromium/chrome/renderer/safe_browsing/phishing_url_feature_extractor_unittest.cc @@ -6,6 +6,8 @@ #include <string> #include <vector> +#include "base/format_macros.h" +#include "base/strings/stringprintf.h" #include "chrome/renderer/safe_browsing/features.h" #include "chrome/renderer/safe_browsing/test_utils.h" #include "testing/gmock/include/gmock/gmock.h" @@ -25,10 +27,24 @@ class PhishingUrlFeatureExtractorTest : public ::testing::Test { PhishingUrlFeatureExtractor::SplitStringIntoLongAlphanumTokens(full, tokens); } + + void FillFeatureMap(size_t count, FeatureMap* features) { + for (size_t i = 0; i < count; ++i) { + EXPECT_TRUE( + features->AddBooleanFeature(base::StringPrintf("Feature%" PRIuS, i))); + } + } }; TEST_F(PhishingUrlFeatureExtractorTest, ExtractFeatures) { std::string url = "http://123.0.0.1/mydocuments/a.file.html"; + FeatureMap features; + + // If feature map is already full, features cannot be extracted. + FillFeatureMap(FeatureMap::kMaxFeatureMapSize, &features); + ASSERT_FALSE(extractor_.ExtractFeatures(GURL(url), &features)); + features.Clear(); + FeatureMap expected_features; expected_features.AddBooleanFeature(features::kUrlHostIsIpAddress); expected_features.AddBooleanFeature(features::kUrlPathToken + @@ -38,9 +54,13 @@ TEST_F(PhishingUrlFeatureExtractorTest, ExtractFeatures) { expected_features.AddBooleanFeature(features::kUrlPathToken + std::string("html")); - FeatureMap features; ASSERT_TRUE(extractor_.ExtractFeatures(GURL(url), &features)); ExpectFeatureMapsAreEqual(features, expected_features); + // If feature map is already full, features cannot be extracted. + features.Clear(); + FillFeatureMap(FeatureMap::kMaxFeatureMapSize - 1, &features); + ASSERT_FALSE(extractor_.ExtractFeatures(GURL(url), &features)); + features.Clear(); url = "http://www.www.cnn.co.uk/sports/sports/index.html?shouldnotappear"; expected_features.Clear(); @@ -61,6 +81,10 @@ TEST_F(PhishingUrlFeatureExtractorTest, ExtractFeatures) { features.Clear(); ASSERT_TRUE(extractor_.ExtractFeatures(GURL(url), &features)); ExpectFeatureMapsAreEqual(features, expected_features); + features.Clear(); + // If feature map is already full, features cannot be extracted. + FillFeatureMap(FeatureMap::kMaxFeatureMapSize - 5, &features); + ASSERT_FALSE(extractor_.ExtractFeatures(GURL(url), &features)); url = "http://justadomain.com/"; expected_features.Clear(); @@ -72,6 +96,10 @@ TEST_F(PhishingUrlFeatureExtractorTest, ExtractFeatures) { features.Clear(); ASSERT_TRUE(extractor_.ExtractFeatures(GURL(url), &features)); ExpectFeatureMapsAreEqual(features, expected_features); + // If feature map is already full, features cannot be extracted. + features.Clear(); + FillFeatureMap(FeatureMap::kMaxFeatureMapSize - 1, &features); + ASSERT_FALSE(extractor_.ExtractFeatures(GURL(url), &features)); url = "http://witharef.com/#abc"; expected_features.Clear(); @@ -96,6 +124,10 @@ TEST_F(PhishingUrlFeatureExtractorTest, ExtractFeatures) { features.Clear(); ASSERT_TRUE(extractor_.ExtractFeatures(GURL(url), &features)); ExpectFeatureMapsAreEqual(features, expected_features); + // If feature map is already full, features cannot be extracted. + features.Clear(); + FillFeatureMap(FeatureMap::kMaxFeatureMapSize - 2, &features); + ASSERT_FALSE(extractor_.ExtractFeatures(GURL(url), &features)); url = "http://unrecognized.tld/"; EXPECT_FALSE(extractor_.ExtractFeatures(GURL(url), &features)); diff --git a/chromium/chrome/renderer/safe_browsing/scorer.cc b/chromium/chrome/renderer/safe_browsing/scorer.cc index 8e37a934f9a..fc592db98c5 100644 --- a/chromium/chrome/renderer/safe_browsing/scorer.cc +++ b/chromium/chrome/renderer/safe_browsing/scorer.cc @@ -10,10 +10,10 @@ #include <unordered_map> #include <unordered_set> -#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "base/strings/string_piece.h" #include "chrome/renderer/safe_browsing/features.h" +#include "components/safe_browsing/content/password_protection/visual_utils.h" #include "components/safe_browsing/core/proto/client_model.pb.h" namespace { @@ -57,14 +57,12 @@ Scorer::~Scorer() {} Scorer* Scorer::Create(const base::StringPiece& model_str) { std::unique_ptr<Scorer> scorer(new Scorer()); ClientSideModel& model = scorer->model_; + // Parse the phishing model. if (!model.ParseFromArray(model_str.data(), model_str.size())) { - DLOG(ERROR) << "Unable to parse phishing model. This Scorer object is " - << "invalid."; RecordScorerCreationStatus(SCORER_FAIL_MODEL_PARSE_ERROR); return NULL; } else if (!model.IsInitialized()) { - DLOG(ERROR) << "Unable to parse phishing model. The model is missing " - << "some required fields. Maybe the .proto file changed?"; + // The model may be missing some required fields. RecordScorerCreationStatus(SCORER_FAIL_MODEL_MISSING_FIELDS); return NULL; } @@ -86,6 +84,32 @@ double Scorer::ComputeScore(const FeatureMap& features) const { return LogOdds2Prob(logodds); } +bool Scorer::GetMatchingVisualTargets(const SkBitmap& bitmap, + ClientPhishingRequest* request) const { + bool has_match = false; + for (const VisualTarget& target : model_.vision_model().targets()) { + base::Optional<VisionMatchResult> result = + visual_utils::IsVisualMatch(bitmap, target); + if (result.has_value()) { + *request->add_vision_match() = result.value(); + has_match = true; + } + } + + if (model_.has_vision_model()) { + // Populate these fields for telementry purposes. They will be filtered in + // the browser process if they are not needed. + VisualFeatures::BlurredImage blurred_image; + if (visual_utils::GetBlurredImage(bitmap, &blurred_image)) { + request->set_screenshot_phash( + visual_utils::GetHashFromBlurredImage(blurred_image)); + request->set_phash_dimension_size(48); + } + } + + return has_match; +} + int Scorer::model_version() const { return model_.version(); } diff --git a/chromium/chrome/renderer/safe_browsing/scorer.h b/chromium/chrome/renderer/safe_browsing/scorer.h index 5fda83d4817..597b4cf047b 100644 --- a/chromium/chrome/renderer/safe_browsing/scorer.h +++ b/chromium/chrome/renderer/safe_browsing/scorer.h @@ -23,6 +23,7 @@ #include "base/macros.h" #include "base/strings/string_piece.h" #include "components/safe_browsing/core/proto/client_model.pb.h" +#include "third_party/skia/include/core/SkBitmap.h" namespace safe_browsing { class FeatureMap; @@ -41,6 +42,11 @@ class Scorer { // (range is inclusive on both ends). virtual double ComputeScore(const FeatureMap& features) const; + // This method matches the given |bitmap| against the visual model. It returns + // true if any visual target matches, and populates |request| appropriately. + virtual bool GetMatchingVisualTargets(const SkBitmap& bitmap, + ClientPhishingRequest* request) const; + // Returns the version number of the loaded client model. int model_version() const; diff --git a/chromium/chrome/renderer/safe_browsing/scorer_unittest.cc b/chromium/chrome/renderer/safe_browsing/scorer_unittest.cc index 9ea41fd1ccc..9083eca05a9 100644 --- a/chromium/chrome/renderer/safe_browsing/scorer_unittest.cc +++ b/chromium/chrome/renderer/safe_browsing/scorer_unittest.cc @@ -59,9 +59,39 @@ class PhishingScorerTest : public ::testing::Test { model_.set_murmur_hash_seed(12345U); model_.set_max_shingles_per_page(10); model_.set_shingle_size(3); + + // The first target hash is all 1-bits, except the first 8. + std::vector<unsigned char> target_hash; + target_hash.push_back('\x30'); + for (int i = 0; i < 288; i++) + target_hash.push_back('\xff'); + target_hash[1] = '\x00'; + VisualTarget* target1 = model_.mutable_vision_model()->add_targets(); + target1->set_digest("target1"); + target1->set_hash(target_hash.data(), target_hash.size()); + target1->mutable_match_config()->add_match_rule()->set_hash_distance(8.0); + + // The second target hash is all 1-bits, except the second 8. + target_hash[1] = '\xff'; + target_hash[2] = '\x00'; + VisualTarget* target2 = model_.mutable_vision_model()->add_targets(); + target2->set_digest("target2"); + target2->set_hash(target_hash.data(), target_hash.size()); + target2->mutable_match_config()->add_match_rule()->set_hash_distance(8.0); + + // Allocate a bitmap for testing visual scoring + sk_sp<SkColorSpace> rec2020 = SkColorSpace::MakeRGB( + {2.22222f, 0.909672f, 0.0903276f, 0.222222f, 0.0812429f, 0, 0}, + SkNamedGamut::kRec2020); + SkImageInfo bitmap_info = + SkImageInfo::Make(1000, 1000, SkColorType::kRGBA_8888_SkColorType, + SkAlphaType::kUnpremul_SkAlphaType, rec2020); + + ASSERT_TRUE(bitmap_.tryAllocPixels(bitmap_info)); } ClientSideModel model_; + SkBitmap bitmap_; }; TEST_F(PhishingScorerTest, HasValidModel) { @@ -145,4 +175,48 @@ TEST_F(PhishingScorerTest, ComputeScore) { EXPECT_TRUE(features.AddBooleanFeature("feature2")); EXPECT_DOUBLE_EQ(0.77729986117469119, scorer->ComputeScore(features)); } + +TEST_F(PhishingScorerTest, GetMatchingVisualTargetsMatchOne) { + std::unique_ptr<Scorer> scorer(Scorer::Create(model_.SerializeAsString())); + + // Make the whole image white + for (int x = 0; x < 1000; x++) + for (int y = 0; y < 1000; y++) + *bitmap_.getAddr32(x, y) = 0xffffffff; + + // Make the first 164 pixels black. This will make the first 8 bits of the + // hash 0. + for (int x = 0; x < 164; x++) + *bitmap_.getAddr32(x, 0) = 0xff000000; + + ClientPhishingRequest request; + scorer->GetMatchingVisualTargets(bitmap_, &request); + ASSERT_EQ(request.vision_match_size(), 1); + EXPECT_EQ(request.vision_match(0).matched_target_digest(), "target1"); +} + +TEST_F(PhishingScorerTest, GetMatchingVisualTargetsMatchBoth) { + std::unique_ptr<Scorer> scorer(Scorer::Create(model_.SerializeAsString())); + + // Make the whole image white + for (int x = 0; x < 1000; x++) + for (int y = 0; y < 1000; y++) + *bitmap_.getAddr32(x, y) = 0xffffffff; + + // Create an alternating black/white pattern to match both targets. The + // pattern is 84 black pixels, then 84 white, then 84 black, then 84 white. + // This causes the hash to start 0F0F, for a distance of 8 from both targets. + for (int x = 0; x < 84; x++) + *bitmap_.getAddr32(x, 0) = 0xff000000; + + for (int x = 168; x < 248; x++) + *bitmap_.getAddr32(x, 0) = 0xff000000; + + ClientPhishingRequest request; + scorer->GetMatchingVisualTargets(bitmap_, &request); + ASSERT_EQ(request.vision_match_size(), 2); + EXPECT_EQ(request.vision_match(0).matched_target_digest(), "target1"); + EXPECT_EQ(request.vision_match(1).matched_target_digest(), "target2"); +} + } // namespace safe_browsing diff --git a/chromium/chrome/renderer/searchbox/searchbox.cc b/chromium/chrome/renderer/searchbox/searchbox.cc index 1690ac1dd92..7439d1f0d1e 100644 --- a/chromium/chrome/renderer/searchbox/searchbox.cc +++ b/chromium/chrome/renderer/searchbox/searchbox.cc @@ -622,8 +622,7 @@ GURL SearchBox::GetURLForMostVisitedItem(InstantRestrictedID item_id) const { return GetMostVisitedItemWithID(item_id, &item) ? item.url : GURL(); } -void SearchBox::DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) { +void SearchBox::DidCommitProvisionalLoad(ui::PageTransition transition) { can_run_js_in_renderframe_ = true; } diff --git a/chromium/chrome/renderer/searchbox/searchbox.h b/chromium/chrome/renderer/searchbox/searchbox.h index c1a8c92c8df..cf274ef90ad 100644 --- a/chromium/chrome/renderer/searchbox/searchbox.h +++ b/chromium/chrome/renderer/searchbox/searchbox.h @@ -242,8 +242,7 @@ class SearchBox : public content::RenderFrameObserver, private: // Overridden from content::RenderFrameObserver: - void DidCommitProvisionalLoad(bool is_same_document_navigation, - ui::PageTransition transition) override; + void DidCommitProvisionalLoad(ui::PageTransition transition) override; void OnDestruct() override; // Overridden from search::mojom::EmbeddedSearchClient: diff --git a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.cc b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.cc index 9af76f69eeb..bde390d6af5 100644 --- a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.cc +++ b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.cc @@ -92,6 +92,10 @@ void SubresourceRedirectHintsAgent::RecordMetricsOnLoadFinished( RecordMetrics(content_length, redirect_result); } +void SubresourceRedirectHintsAgent::ClearImageHints() { + public_image_urls_.clear(); +} + void SubresourceRedirectHintsAgent::RecordMetrics( int64_t content_length, RedirectResult redirect_result) const { diff --git a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h index b38d98d599a..76448440f3a 100644 --- a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h +++ b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h @@ -8,6 +8,7 @@ #include "base/containers/flat_set.h" #include "base/macros.h" #include "base/timer/timer.h" +#include "chrome/common/subresource_redirect_service.mojom.h" #include "third_party/blink/public/mojom/loader/previews_resource_loading_hints.mojom.h" #include "url/gurl.h" @@ -68,6 +69,9 @@ class SubresourceRedirectHintsAgent { int64_t content_length, RedirectResult redirect_result); + // Clears the image hint urls. + void ClearImageHints(); + private: void OnHintsReceiveTimeout(); void RecordMetrics(int64_t content_length, diff --git a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.cc b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.cc index f4c808f42c2..8ce4ae827f9 100644 --- a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.cc +++ b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.cc @@ -7,6 +7,7 @@ #include "base/memory/ptr_util.h" #include "base/metrics/field_trial_params.h" #include "base/metrics/histogram_macros.h" +#include "base/time/time.h" #include "chrome/renderer/previews/resource_loading_hints_agent.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_params.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_util.h" @@ -16,6 +17,7 @@ #include "net/base/escape.h" #include "net/base/load_flags.h" #include "net/http/http_status_code.h" +#include "net/http/http_util.h" #include "services/network/public/mojom/fetch_api.mojom-shared.h" #include "services/network/public/mojom/url_response_head.mojom.h" #include "third_party/blink/public/common/features.h" @@ -44,6 +46,13 @@ bool ShouldCompressionServerRedirectSubresource() { "enable_subresource_server_redirect", false); } +base::TimeDelta GetCompressionRedirectTimeout() { + return base::TimeDelta::FromMilliseconds( + base::GetFieldTrialParamByFeatureAsInt( + blink::features::kSubresourceRedirect, "subresource_redirect_timeout", + 5000)); +} + } // namespace // static @@ -117,16 +126,29 @@ void SubresourceRedirectURLLoaderThrottle::WillStartRequest( request->url = GetSubresourceURLForURL(request->url); did_redirect_compressed_origin_ = true; *defer = false; + + DCHECK(!redirect_timeout_timer_); + redirect_timeout_timer_ = std::make_unique<base::OneShotTimer>(); + redirect_timeout_timer_->Start( + FROM_HERE, GetCompressionRedirectTimeout(), + base::BindOnce(&SubresourceRedirectURLLoaderThrottle::OnRedirectTimeout, + base::Unretained(this))); } -SubresourceRedirectHintsAgent* -SubresourceRedirectURLLoaderThrottle::GetSubresourceRedirectHintsAgent() { +previews::ResourceLoadingHintsAgent* +SubresourceRedirectURLLoaderThrottle::GetResourceLoadingHintsAgent() { + // The ResourceLoadingHintsAgent is main-frame only. if (content::RenderFrame* render_frame = content::RenderFrame::FromRoutingID(render_frame_id_)) { - if (auto* resource_loading_hints_agent = - previews::ResourceLoadingHintsAgent::Get(render_frame)) { - return &resource_loading_hints_agent->subresource_redirect_hints_agent(); - } + return previews::ResourceLoadingHintsAgent::Get(render_frame); + } + return nullptr; +} + +SubresourceRedirectHintsAgent* +SubresourceRedirectURLLoaderThrottle::GetSubresourceRedirectHintsAgent() { + if (auto* resource_loading_hints_agent = GetResourceLoadingHintsAgent()) { + return &resource_loading_hints_agent->subresource_redirect_hints_agent(); } return nullptr; } @@ -138,6 +160,12 @@ void SubresourceRedirectURLLoaderThrottle::WillRedirectRequest( std::vector<std::string>* to_be_removed_request_headers, net::HttpRequestHeaders* modified_request_headers, net::HttpRequestHeaders* modified_cors_exempt_request_headers) { + if (did_redirect_compressed_origin_ && redirect_timeout_timer_) { + redirect_timeout_timer_->Start( + FROM_HERE, GetCompressionRedirectTimeout(), + base::BindOnce(&SubresourceRedirectURLLoaderThrottle::OnRedirectTimeout, + base::Unretained(this))); + } UMA_HISTOGRAM_ENUMERATION( "SubresourceRedirect.CompressionAttempt.ResponseCode", static_cast<net::HttpStatusCode>(response_head.headers->response_code()), @@ -160,6 +188,7 @@ void SubresourceRedirectURLLoaderThrottle::BeforeWillProcessResponse( "SubresourceRedirect.CompressionAttempt.ResponseCode", static_cast<net::HttpStatusCode>(response_head.headers->response_code()), net::HTTP_VERSION_NOT_SUPPORTED); + redirect_timeout_timer_.reset(); // Do nothing with 2XX responses, as these requests were handled // correctly by the compression server. @@ -171,6 +200,24 @@ void SubresourceRedirectURLLoaderThrottle::BeforeWillProcessResponse( redirect_result_ = SubresourceRedirectHintsAgent::RedirectResult::kIneligibleOtherImage; + // 503 response code indicates loadshed from the compression server. Notify + // the browser process which will bypass subresource redirect for subsequent + // page loads. Retry-After response header may mention the bypass duration, + // otherwise the browser will choose a random duration. + if (response_head.headers->response_code() == 503) { + std::string retry_after_string; + base::TimeDelta retry_after; + if (response_head.headers->EnumerateHeader(nullptr, "Retry-After", + &retry_after_string)) { + net::HttpUtil::ParseRetryAfterHeader(retry_after_string, + base::Time::Now(), &retry_after); + } + if (auto* resource_loading_hints_agent = GetResourceLoadingHintsAgent()) { + resource_loading_hints_agent->NotifyHttpsImageCompressionFetchFailed( + retry_after); + } + } + // Non 2XX responses from the compression server need to have unaltered // requests sent to the original resource. did_redirect_compressed_origin_ = false; @@ -236,11 +283,25 @@ void SubresourceRedirectURLLoaderThrottle::WillOnCompleteWithError( // If the server fails, restart the request to the original resource, and // record it. did_redirect_compressed_origin_ = false; + redirect_timeout_timer_.reset(); delegate_->RestartWithURLResetAndFlags(net::LOAD_NORMAL); UMA_HISTOGRAM_BOOLEAN( "SubresourceRedirect.CompressionAttempt.ServerResponded", false); } +void SubresourceRedirectURLLoaderThrottle::OnRedirectTimeout() { + DCHECK(did_redirect_compressed_origin_); + did_redirect_compressed_origin_ = false; + delegate_->RestartWithURLResetAndFlagsNow(net::LOAD_NORMAL); + if (auto* resource_loading_hints_agent = GetResourceLoadingHintsAgent()) { + resource_loading_hints_agent->NotifyHttpsImageCompressionFetchFailed( + base::TimeDelta()); + resource_loading_hints_agent->subresource_redirect_hints_agent() + .ClearImageHints(); + } + UMA_HISTOGRAM_BOOLEAN("SubresourceRedirect.CompressionFetchTimeout", true); +} + void SubresourceRedirectURLLoaderThrottle::DetachFromCurrentSequence() {} } // namespace subresource_redirect diff --git a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h index 3c02b9595df..e5f8a1d0db6 100644 --- a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h +++ b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h @@ -6,6 +6,7 @@ #define CHROME_RENDERER_SUBRESOURCE_REDIRECT_SUBRESOURCE_REDIRECT_URL_LOADER_THROTTLE_H_ #include "base/macros.h" +#include "base/timer/timer.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h" #include "third_party/blink/public/common/loader/url_loader_throttle.h" @@ -13,6 +14,10 @@ namespace blink { class WebURLRequest; } // namespace blink +namespace previews { +class ResourceLoadingHintsAgent; +} // namespace previews + namespace subresource_redirect { class SubresourceRedirectHintsAgent; @@ -27,6 +32,8 @@ class SubresourceRedirectURLLoaderThrottle : public blink::URLLoaderThrottle { ~SubresourceRedirectURLLoaderThrottle() override; + previews::ResourceLoadingHintsAgent* GetResourceLoadingHintsAgent(); + // virtual for testing. virtual SubresourceRedirectHintsAgent* GetSubresourceRedirectHintsAgent(); @@ -58,6 +65,9 @@ class SubresourceRedirectURLLoaderThrottle : public blink::URLLoaderThrottle { SubresourceRedirectURLLoaderThrottle(int render_frame_id, bool allowed_to_redirect); + // Callback invoked when the redirect fetch times out. + void OnRedirectTimeout(); + // Render frame id to get the hints agent of the render frame. const int render_frame_id_; @@ -72,6 +82,9 @@ class SubresourceRedirectURLLoaderThrottle : public blink::URLLoaderThrottle { // itself is compressed origin. bool did_redirect_compressed_origin_ = false; + // Timer to detect whether the response from compression server has timed out. + std::unique_ptr<base::OneShotTimer> redirect_timeout_timer_; + DISALLOW_COPY_AND_ASSIGN(SubresourceRedirectURLLoaderThrottle); }; diff --git a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle_unittest.cc b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle_unittest.cc index 8c476608f2f..7d4b7278a25 100644 --- a/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle_unittest.cc +++ b/chromium/chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle_unittest.cc @@ -5,6 +5,7 @@ #include "chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h" #include "base/test/scoped_feature_list.h" +#include "base/test/task_environment.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_hints_agent.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_util.h" #include "content/public/common/previews_state.h" @@ -36,6 +37,7 @@ class TestSubresourceRedirectURLLoaderThrottle } private: + base::test::SingleThreadTaskEnvironment task_environment_; SubresourceRedirectHintsAgent subresource_redirect_hints_agent_; }; diff --git a/chromium/chrome/renderer/url_loader_throttle_provider_impl.cc b/chromium/chrome/renderer/url_loader_throttle_provider_impl.cc index e7dc7dc02e2..8a4536d5f7c 100644 --- a/chromium/chrome/renderer/url_loader_throttle_provider_impl.cc +++ b/chromium/chrome/renderer/url_loader_throttle_provider_impl.cc @@ -16,6 +16,7 @@ #include "chrome/renderer/chrome_content_renderer_client.h" #include "chrome/renderer/chrome_render_frame_observer.h" #include "chrome/renderer/chrome_render_thread_observer.h" +#include "chrome/renderer/lite_video/lite_video_url_loader_throttle.h" #include "chrome/renderer/prerender/prerender_helper.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_params.h" #include "chrome/renderer/subresource_redirect/subresource_redirect_url_loader_throttle.h" @@ -200,6 +201,7 @@ URLLoaderThrottleProviderImpl::CreateThrottles( throttles.push_back(std::make_unique<GoogleURLLoaderThrottle>( #if defined(OS_ANDROID) client_data_header, + /* night_mode_enabled= */ false, #endif ChromeRenderThreadObserver::GetDynamicParams())); @@ -214,6 +216,13 @@ URLLoaderThrottleProviderImpl::CreateThrottles( if (throttle) throttles.push_back(std::move(throttle)); + if (render_frame_id != MSG_ROUTING_NONE) { + auto throttle = lite_video::LiteVideoURLLoaderThrottle::MaybeCreateThrottle( + request, render_frame_id); + if (throttle) + throttles.push_back(std::move(throttle)); + } + return throttles; } |
